BitMagic-C++
bmfunc.h
Go to the documentation of this file.
1 #ifndef BMFUNC__H__INCLUDED__
2 #define BMFUNC__H__INCLUDED__
3 /*
4 Copyright(c) 2002-2017 Anatoliy Kuznetsov(anatoliy_kuznetsov at yahoo.com)
5 
6 Licensed under the Apache License, Version 2.0 (the "License");
7 you may not use this file except in compliance with the License.
8 You may obtain a copy of the License at
9 
10  http://www.apache.org/licenses/LICENSE-2.0
11 
12 Unless required by applicable law or agreed to in writing, software
13 distributed under the License is distributed on an "AS IS" BASIS,
14 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 See the License for the specific language governing permissions and
16 limitations under the License.
17 
18 For more information please visit: http://bitmagic.io
19 */
20 
21 /*! \file bmfunc.h
22  \brief Bit manipulation primitives (internal)
23 */
24 
25 #include <memory.h>
26 
27 #include "bmdef.h"
28 #include "bmutil.h"
29 
30 
31 #ifdef _MSC_VER
32 # pragma warning( disable: 4146 )
33 #endif
34 
35 namespace bm
36 {
37 
38 
39 inline
41  bm::word_t left,
42  bm::word_t right) BMNOEXCEPT;
43 
44 inline
46  bm::word_t left,
47  bm::word_t right) BMNOEXCEPT;
48 
49 /*!
50  @brief Structure with statistical information about memory
51  allocation footprint, serialization projection, number of vectors
52  @ingroup bvector
53 */
55 {
56  size_t bit_blocks; ///< Number of bit blocks
57  size_t gap_blocks; ///< Number of GAP blocks
58  size_t ptr_sub_blocks; ///< Number of sub-blocks
59  size_t bv_count; ///< Number of bit-vectors
60  size_t max_serialize_mem; ///< estimated maximum memory for serialization
61  size_t memory_used; ///< memory usage for all blocks and service tables
62  size_t gap_cap_overhead; ///< gap memory overhead between length and capacity
63  gap_word_t gap_levels[bm::gap_levels]; ///< GAP block lengths in the bvect
64  unsigned long long gaps_by_level[bm::gap_levels]; ///< number of GAP blocks at each level
65 
66  /// cound bit block
68  {
69  ++bit_blocks;
70  size_t mem_used = sizeof(bm::word_t) * bm::set_block_size;
71  memory_used += mem_used;
72  max_serialize_mem += mem_used;
73  }
74 
75  /// count gap block
76  void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
77  {
78  ++gap_blocks;
79  size_t mem_used = (capacity * sizeof(gap_word_t));
80  memory_used += mem_used;
81  max_serialize_mem += (unsigned)(length * sizeof(gap_word_t));
82  BM_ASSERT(length <= capacity);
83  gap_cap_overhead += (capacity - length) * sizeof(gap_word_t);
84  for (unsigned i = 0; i < bm::gap_levels; ++i)
85  {
86  if (capacity == gap_levels[i])
87  {
88  gaps_by_level[i]++;
89  return;
90  }
91  }
92  BM_ASSERT(0);
93  }
94 
95  /// Reset statisctics
97  {
100  for (unsigned i = 0; i < bm::gap_levels; ++i)
101  gaps_by_level[i] = 0ull;
102  }
103 
104  /// Sum data from another sttructure
105  void add(const bv_statistics& st) BMNOEXCEPT
106  {
107  bit_blocks += st.bit_blocks;
108  gap_blocks += st.gap_blocks;
109  ptr_sub_blocks += st.ptr_sub_blocks;
110  bv_count += st.bv_count;
111  max_serialize_mem += st.max_serialize_mem + 8;
112  memory_used += st.memory_used;
113  gap_cap_overhead += st.gap_cap_overhead;
114  }
115 };
116 
117 /**
118  @brief Pair type
119 */
120 template<typename First, typename Second>
121 struct pair
122 {
123  First first;
124  Second second;
125 
126  pair(First f, Second s) : first(f), second(s) {}
127 };
128 
129 /**
130  \brief bit-decode cache structure
131 */
133 {
134  unsigned short bits[65]; //< decoded bits
135  unsigned bcnt; //< length of bits array
136  bm::id64_t cvalue; //< cache decoded value
137 
139 };
140 
141 
142 /**
143  \brief Recalc linear bvector block index into 2D matrix coordinates
144  \internal
145 */
146 template<typename BI_TYPE>
147 void get_block_coord(BI_TYPE nb, unsigned& i, unsigned& j) BMNOEXCEPT
148 {
149  i = unsigned(nb >> bm::set_array_shift); // top block address
150  j = unsigned(nb & bm::set_array_mask); // address in sub-block
151 }
152 
153 /**
154  Compute bit address of the first bit in a superblock
155  \internal
156 */
157 template<typename RTYPE>
159 {
160  return RTYPE(i) * bm::set_sub_total_bits;
161 }
162 
163 /**
164  Compute bit address of the first bit in a block
165  \internal
166 */
167 template<typename RTYPE>
168 BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
169 {
170  RTYPE idx = bm::get_super_block_start<RTYPE>(i);
171  idx += (j) * bm::gap_max_bits;
172  return idx;
173 }
174 
175 /*!
176  @defgroup gapfunc GAP functions
177  GAP functions implement different opereations on GAP compressed blocks (internals)
178  and serve as a minimal building blocks.
179  \internal
180  @ingroup bvector
181  */
182 
183 /*!
184  @defgroup bitfunc BIT functions
185  Bit functions implement different opereations on bit blocks (internals)
186  and serve as a minimal building blocks.
187  \internal
188  @ingroup bvector
189  */
190 
191 
192 /*!
193  Returns bit count
194  @ingroup bitfunc
195 */
198 {
199 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
200  return bm::id_t(_mm_popcnt_u32(w));
201 #else
202  #if defined(BM_USE_GCC_BUILD)
203  return (bm::id_t)__builtin_popcount(w);
204  #else
205  return
206  bm::bit_count_table<true>::_count[(unsigned char)(w)] +
207  bm::bit_count_table<true>::_count[(unsigned char)((w) >> 8)] +
208  bm::bit_count_table<true>::_count[(unsigned char)((w) >> 16)] +
209  bm::bit_count_table<true>::_count[(unsigned char)((w) >> 24)];
210  #endif
211 #endif
212 }
213 
214 inline
215 int parallel_popcnt_32(unsigned int n) BMNOEXCEPT
216 {
217  unsigned int tmp;
218 
219  tmp = n - ((n >> 1) & 033333333333)
220  - ((n >> 2) & 011111111111);
221  return ((tmp + (tmp >> 3)) & 030707070707) % 63;
222 }
223 
224 
225 /*!
226  Function calculates number of 1 bits in 64-bit word.
227  @ingroup bitfunc
228 */
231 {
232 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
233 #if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
234  return unsigned(_mm_popcnt_u64(x));
235 #else
236  return _mm_popcnt_u32(x >> 32) + _mm_popcnt_u32((unsigned)x);
237 #endif
238 #else
239  #if defined(BM_USE_GCC_BUILD)
240  return (unsigned)__builtin_popcountll(x);
241  #else
242  x = x - ((x >> 1) & 0x5555555555555555);
243  x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);
244  x = (x + (x >> 4)) & 0x0F0F0F0F0F0F0F0F;
245  x = x + (x >> 8);
246  x = x + (x >> 16);
247  x = x + (x >> 32);
248  return x & 0xFF;
249  #endif
250 #endif
251 }
252 
253 inline
256 {
257  const bm::id64_t m1 = 0x5555555555555555U;
258  const bm::id64_t m2 = 0x3333333333333333U;
259  const bm::id64_t m3 = 0x0F0F0F0F0F0F0F0FU;
260  const bm::id64_t m4 = 0x000000FF000000FFU;
261 
262  x = x - ((x >> 1) & m1);
263  y = y - ((y >> 1) & m1);
264  u = u - ((u >> 1) & m1);
265  v = v - ((v >> 1) & m1);
266  x = (x & m2) + ((x >> 2) & m2);
267  y = (y & m2) + ((y >> 2) & m2);
268  u = (u & m2) + ((u >> 2) & m2);
269  v = (v & m2) + ((v >> 2) & m2);
270  x = x + y;
271  u = u + v;
272  x = (x & m3) + ((x >> 4) & m3);
273  u = (u & m3) + ((u >> 4) & m3);
274  x = x + u;
275  x = x + (x >> 8);
276  x = x + (x >> 16);
277  x = x & m4;
278  x = x + (x >> 32);
279  return x & 0x000001FFU;
280 }
281 
282 
283 // --------------------------------------------------------------
284 // Functions for bit-scanning
285 // --------------------------------------------------------------
286 
287 /*!
288  \brief Templated algorithm to unpacks octet based word into list of ON bit indexes
289  \param w - value
290  \param func - bit functor
291 
292  @ingroup bitfunc
293 */
294 template<typename T, typename F>
295 void bit_for_each_4(T w, F& func)
296 {
297  for (unsigned sub_octet = 0; w != 0; w >>= 4, sub_octet += 4)
298  {
299  switch (w & 15) // 1111
300  {
301  case 0: // 0000
302  break;
303  case 1: // 0001
304  func(sub_octet);
305  break;
306  case 2: // 0010
307  func(sub_octet + 1);
308  break;
309  case 3: // 0011
310  func(sub_octet, sub_octet + 1);
311  break;
312  case 4: // 0100
313  func(sub_octet + 2);
314  break;
315  case 5: // 0101
316  func(sub_octet, sub_octet + 2);
317  break;
318  case 6: // 0110
319  func(sub_octet + 1, sub_octet + 2);
320  break;
321  case 7: // 0111
322  func(sub_octet, sub_octet + 1, sub_octet + 2);
323  break;
324  case 8: // 1000
325  func(sub_octet + 3);
326  break;
327  case 9: // 1001
328  func(sub_octet, sub_octet + 3);
329  break;
330  case 10: // 1010
331  func(sub_octet + 1, sub_octet + 3);
332  break;
333  case 11: // 1011
334  func(sub_octet, sub_octet + 1, sub_octet + 3);
335  break;
336  case 12: // 1100
337  func(sub_octet + 2, sub_octet + 3);
338  break;
339  case 13: // 1101
340  func(sub_octet, sub_octet + 2, sub_octet + 3);
341  break;
342  case 14: // 1110
343  func(sub_octet + 1, sub_octet + 2, sub_octet + 3);
344  break;
345  case 15: // 1111
346  func(sub_octet, sub_octet + 1, sub_octet + 2, sub_octet + 3);
347  break;
348  default:
349  BM_ASSERT(0);
350  break;
351  }
352 
353  } // for
354 }
355 
356 
357 /*!
358  \brief Templated algorithm to unpacks word into list of ON bit indexes
359  \param w - value
360  \param func - bit functor
361 
362  @ingroup bitfunc
363 */
364 template<typename T, typename F>
365 void bit_for_each(T w, F& func)
366 {
367  // Note: 4-bit table method works slower than plain check approach
368  for (unsigned octet = 0; w != 0; w >>= 8, octet += 8)
369  {
370  if (w & 1) func(octet + 0);
371  if (w & 2) func(octet + 1);
372  if (w & 4) func(octet + 2);
373  if (w & 8) func(octet + 3);
374  if (w & 16) func(octet + 4);
375  if (w & 32) func(octet + 5);
376  if (w & 64) func(octet + 6);
377  if (w & 128) func(octet + 7);
378 
379  } // for
380 }
381 
382 /*! @brief Adaptor to copy 1 bits to array
383  @internal
384 */
385 template<typename B>
387 {
388 public:
389  copy_to_array_functor(B* bits): bp_(bits)
390  {}
391 
392  B* ptr() { return bp_; }
393 
394  void operator()(unsigned bit_idx) BMNOEXCEPT { *bp_++ = (B)bit_idx; }
395 
396  void operator()(unsigned bit_idx0,
397  unsigned bit_idx1) BMNOEXCEPT
398  {
399  bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
400  bp_+=2;
401  }
402 
403  void operator()(unsigned bit_idx0,
404  unsigned bit_idx1,
405  unsigned bit_idx2) BMNOEXCEPT
406  {
407  bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1; bp_[2] = (B)bit_idx2;
408  bp_+=3;
409  }
410 
411  void operator()(unsigned bit_idx0,
412  unsigned bit_idx1,
413  unsigned bit_idx2,
414  unsigned bit_idx3) BMNOEXCEPT
415  {
416  bp_[0] = (B)bit_idx0; bp_[1] = (B)bit_idx1;
417  bp_[2] = (B)bit_idx2; bp_[3] = (B)bit_idx3;
418  bp_+=4;
419  }
420 
421 private:
423  copy_to_array_functor& operator=(const copy_to_array_functor&);
424 private:
425  B* bp_;
426 };
427 
428 
429 /*!
430  \brief Unpacks word into list of ON bit indexes
431  \param w - value
432  \param bits - pointer on the result array
433  \return number of bits in the list
434 
435  @ingroup bitfunc
436 */
437 template<typename T,typename B>
438 unsigned bit_list(T w, B* bits) BMNOEXCEPT
439 {
440  copy_to_array_functor<B> func(bits);
441  bit_for_each(w, func);
442  return (unsigned)(func.ptr() - bits);
443 }
444 
445 
446 
447 /*!
448  \brief Unpacks word into list of ON bit indexes (quad-bit based)
449  \param w - value
450  \param bits - pointer on the result array
451  \return number of bits in the list
452 
453  @ingroup bitfunc
454 */
455 template<typename T,typename B>
456 unsigned bit_list_4(T w, B* bits) BMNOEXCEPT
457 {
458  copy_to_array_functor<B> func(bits);
459  bit_for_each_4(w, func);
460  return (unsigned)(func.ptr() - bits);
461 }
462 
463 /*!
464  \brief Unpacks word into list of ON bit indexes using popcnt method
465  \param w - value
466  \param bits - pointer on the result array
467  \param offs - value to add to bit position (programmed shift)
468  \return number of bits in the list
469 
470  @ingroup bitfunc
471  @internal
472 */
473 template<typename B>
474 unsigned short
475 bitscan_popcnt(bm::id_t w, B* bits, unsigned short offs) BMNOEXCEPT
476 {
477  unsigned pos = 0;
478  while (w)
479  {
480  bm::id_t t = w & -w;
481  bits[pos++] = (B)(bm::word_bitcount(t - 1) + offs);
482  w &= w - 1;
483  }
484  return (unsigned short)pos;
485 }
486 
487 /*!
488  \brief Unpacks word into list of ON bit indexes using popcnt method
489  \param w - value
490  \param bits - pointer on the result array
491  \return number of bits in the list
492 
493  @ingroup bitfunc
494  @internal
495 */
496 template<typename B>
497 unsigned short bitscan_popcnt(bm::id_t w, B* bits) BMNOEXCEPT
498 {
499  unsigned pos = 0;
500  while (w)
501  {
502  bm::id_t t = w & -w;
503  bits[pos++] = (B)(bm::word_bitcount(t - 1));
504  w &= w - 1;
505  }
506  return (unsigned short)pos;
507 }
508 
509 
510 /*!
511  \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
512  \param w - value
513  \param bits - pointer on the result array
514  \return number of bits in the list
515  @ingroup bitfunc
516 */
517 template<typename B>
518 unsigned short bitscan_popcnt64(bm::id64_t w, B* bits) BMNOEXCEPT
519 {
520  unsigned short pos = 0;
521  while (w)
522  {
523  bm::id64_t t = bmi_blsi_u64(w); // w & -w;
524  bits[pos++] = (B) bm::word_bitcount64(t - 1);
525  w = bmi_bslr_u64(w); // w &= w - 1;
526  }
527  return pos;
528 }
529 
530 /*!
531  \brief Unpacks 64-bit word into list of ON bit indexes using popcnt method
532  \param w - value
533  \param bits - pointer on the result array
534  \param offs - value to add to bit position (programmed shift)
535  \return number of bits in the list
536  @ingroup bitfunc
537 */
538 template<typename B>
539 unsigned short
540 bitscan_popcnt64(bm::id64_t w, B* bits, unsigned short offs) BMNOEXCEPT
541 {
542  unsigned short pos = 0;
543  while (w)
544  {
545  bm::id64_t t = bmi_blsi_u64(w); // w & -w;
546  bits[pos++] = B(bm::word_bitcount64(t - 1) + offs);
547  w = bmi_bslr_u64(w); // w &= w - 1;
548  }
549  return pos;
550 }
551 
552 
553 template<typename V, typename B>
554 unsigned short bitscan(V w, B* bits) BMNOEXCEPT
555 {
556  if (bm::conditional<sizeof(V) == 8>::test())
557  return bm::bitscan_popcnt64(w, bits);
558  else
559  return bm::bitscan_popcnt((bm::word_t)w, bits);
560 }
561 
562 // --------------------------------------------------------------
563 // Functions for select
564 // --------------------------------------------------------------
565 
566 /**
567  \brief word find index of the rank-th bit set by bit-testing
568  \param w - 64-bit work to search
569  \param rank - rank to select (should be > 0)
570 
571  \return selected value (inxed of bit set)
572 */
573 inline
574 unsigned word_select64_linear(bm::id64_t w, unsigned rank) BMNOEXCEPT
575 {
576  BM_ASSERT(w);
577  BM_ASSERT(rank);
578 
579  for (unsigned count = 0; w; w >>=1ull, ++count)
580  {
581  rank -= unsigned(w & 1ull);
582  if (!rank)
583  return count;
584  }
585  BM_ASSERT(0); // shoud not be here if rank is achievable
586  return ~0u;
587 }
588 
589 /**
590  \brief word find index of the rank-th bit set by bit-testing
591  \param w - 64-bit work to search
592  \param rank - rank to select (should be > 0)
593 
594  \return selected value (inxed of bit set)
595 */
596 inline
597 unsigned word_select64_bitscan(bm::id64_t w, unsigned rank) BMNOEXCEPT
598 {
599  BM_ASSERT(w);
600  BM_ASSERT(rank);
601  BM_ASSERT(rank <= bm::word_bitcount64(w));
602 
603  do
604  {
605  --rank;
606  if (!rank)
607  break;
608  w &= w - 1;
609  } while (1);
610 
611  bm::id64_t t = w & -w;
612  unsigned count = bm::word_bitcount64(t - 1);
613  return count;
614 }
615 
616 /**
617  \brief word find index of the rank-th bit set by bit-testing
618  \param w - 64-bit work to search
619  \param rank - rank to select (should be > 0)
620 
621  \return selected value (inxed of bit set)
622 */
623 inline
624 unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
625 {
626 #if defined(BMI2_SELECT64)
627  return BMI2_SELECT64(w, rank);
628 #else
629  #if defined(BMI1_SELECT64)
630  return BMI2_SELECT64(w, rank);
631  #else
632  return bm::word_select64_bitscan(w, rank);
633  #endif
634 #endif
635 }
636 
637 // --------------------------------------------------------------
638 // Functions for bit-block digest calculation
639 // --------------------------------------------------------------
640 
641 /*!
642  \brief Compute digest mask for word address in block
643 
644  @return digest bit-mask
645 
646  @ingroup bitfunc
647  @internal
648 */
651 {
652  bm::id64_t mask(1ull);
653  return mask << (w_idx / bm::set_block_digest_wave_size);
654 }
655 
656 /**
657  \brief Compute digest mask for [from..to] positions
658  \param from - range from (in bit-block coordinates)
659  \param to - range to (in bit-block coordinates)
660 
661  @ingroup bitfunc
662  @internal
663 */
665 bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
666 {
667  BM_ASSERT(from <= to);
668 
669  bm::id64_t digest_from = from >> bm::set_block_digest_pos_shift;
670  bm::id64_t digest_to = to >> bm::set_block_digest_pos_shift;;
671  const bm::id64_t maskFF(~0ull);
672  return ((maskFF) >> (63 - (digest_to - digest_from))) << digest_from;
673 }
674 
675 /*!
676  \brief check if all digest bits for the range [from..to] are 0
677 
678  \param digest - current digest
679  \param bitpos_from - range from (in bit-block coordinates)
680  \param bitpos_to - range to (in bit-block coordinates)
681 
682  @return true if all range is zero
683 
684  @ingroup bitfunc
685  @internal
686 */
687 inline
689  unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
690 {
691  bm::id64_t mask = bm::digest_mask(bitpos_from, bitpos_to);
692  return !(digest & mask);
693 }
694 
695 /*!
696  \brief Init block with 000111000 pattren based on digest
697  \param block - Bit block [out]
698  \param digest - digest (used for block initialization)
699 
700  @ingroup bitfunc
701  @internal
702 */
703 inline
705 {
706  unsigned off;
707  for (unsigned i = 0; i < 64; ++i)
708  {
710  bm::word_t mask = (digest & 1) ? ~0u : 0u;
711 #if defined(VECT_BLOCK_SET_DIGEST)
712  VECT_BLOCK_SET_DIGEST(&block[off], mask);
713 #else
714  for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
715  {
716  block[off+j+0] = block[off+j+1] =
717  block[off+j+2] = block[off+j+3] = mask;
718  } // for j
719 #endif
720  digest >>= 1ull;
721  } // for
722 }
723 
724 /*!
725  \brief Compute digest for 64 non-zero areas
726  \param block - Bit block
727 
728  @return digest bit-mask (0 means all block is empty)
729 
730  @ingroup bitfunc
731  @internal
732 */
733 inline
735 {
736  bm::id64_t digest0 = 0;
737  unsigned off;
738 
739  for (unsigned i = 0; i < 64; ++i)
740  {
742  #if defined(VECT_IS_DIGEST_ZERO)
743  bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
744  digest0 |= bm::id64_t(!all_zero) << i;
745  #else
746  const bm::id64_t mask(1ull);
747  for (unsigned j = 0; j < bm::set_block_digest_wave_size; j+=4)
748  {
749  bm::word_t w =
750  block[off+j+0] | block[off+j+1] | block[off+j+2] | block[off+j+3];
751  if (w)
752  {
753  digest0 |= (mask << i);
754  break;
755  }
756  } // for j
757  #endif
758 
759  } // for i
760  return digest0;
761 }
762 
763 /*!
764  \brief Compute digest for 64 non-zero areas based on existing digest
765  (function revalidates zero areas)
766  \param block - bit block
767  \param digest - start digest
768 
769  @return digest bit-mask (0 means all block is empty)
770 
771  @ingroup bitfunc
772  @internal
773 */
774 inline
777 {
778  const bm::id64_t mask(1ull);
779  bm::id64_t d = digest;
780 
781  while (d)
782  {
783  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
784 
785  unsigned wave = bm::word_bitcount64(t - 1);
786  unsigned off = wave * bm::set_block_digest_wave_size;
787 
788  #if defined(VECT_IS_DIGEST_ZERO)
789  bool all_zero = VECT_IS_DIGEST_ZERO(&block[off]);
790  digest &= all_zero ? ~(mask << wave) : digest;
791  #else
793  (const bm::bit_block_t::bunion_t*)(&block[off]);
794  bm::id64_t w64 = 0;
795  unsigned j = 0;
796  do
797  {
798  w64 |=
799  src_u->w64[j+0] | src_u->w64[j+1] | src_u->w64[j+2] | src_u->w64[j+3];
800  j += 4;
801  } while ((j < bm::set_block_digest_wave_size/2) & !w64);
802  digest &= w64 ? digest : ~(mask << wave);
803  #endif
804 
805  d = bm::bmi_bslr_u64(d); // d &= d - 1;
806  } // while
807 
808  BM_ASSERT(bm::calc_block_digest0(block) == digest);
809  return digest;
810 }
811 
812 // --------------------------------------------------------------
813 
814 
815 /// Returns true if set operation is constant (bitcount)
816 inline
818 {
819  return (int(op) >= int(set_COUNT));
820 }
821 
822 /**
823  Convert set operation to operation
824 */
825 inline
827 {
828  BM_ASSERT(op == set_AND ||
829  op == set_OR ||
830  op == set_SUB ||
831  op == set_XOR);
832  return (bm::operation) op;
833 }
834 
835 //---------------------------------------------------------------------
836 
837 /**
838  Structure carries pointer on bit block with all bits 1
839  @ingroup bitfunc
840  @internal
841 */
842 template<bool T> struct all_set
843 {
845  {
849 
851  {
852  ::memset(_p, 0xFF, sizeof(_p)); // set FULL BLOCK content (all 1s)
853  if (bm::conditional<sizeof(void*) == 8>::test())
854  {
855  const unsigned long long magic_mask = 0xFFFFfffeFFFFfffe;
856  ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
857  for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
858  _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
859  }
860  else
861  {
862  const unsigned magic_mask = 0xFFFFfffe;
863  ::memcpy(&_p_fullp, &magic_mask, sizeof(magic_mask));
864  for (unsigned i = 0; i < bm::set_sub_array_size; ++i)
865  _s[i] = reinterpret_cast<bm::word_t*>(magic_mask);
866  }
867  }
868  };
869 
870  // version with minimal branching, super-scalar friendly
871  //
872  inline
874  {
875  bm::id64_t type;
876  if (bm::conditional<sizeof(void*) == 8>::test())
877  {
878  bm::id64_t w = reinterpret_cast<unsigned long long>(bp);
879  type = (w & 3) | // FULL BLOCK or GAP
880  ((bp == _block._p) << 1);
881  type = type ? type : w;
882  }
883  else
884  {
885  unsigned w = reinterpret_cast<unsigned long>(bp);
886  type = (w & 3) | // FULL BLOCK or GAP
887  ((bp == _block._p) << 1);
888  type = type ? type : w;
889  }
890  return type;
891  }
892 
894  static bool is_full_block(const bm::word_t* bp) BMNOEXCEPT
895  { return (bp == _block._p || bp == _block._p_fullp); }
896 
899  { return (bp && !(bp == _block._p || bp == _block._p_fullp)); }
900 
902 };
903 
904 
905 template<bool T> typename all_set<T>::all_set_block all_set<T>::_block;
906 
907 /// XOR swap two scalar variables
908 template<typename W>
909 void xor_swap(W& x, W& y) BMNOEXCEPT
910 {
911  BM_ASSERT(&x != &y);
912  x ^= y;
913  y ^= x;
914  x ^= y;
915 }
916 
917 /*!
918  Fini not NULL position
919  @return index of not NULL pointer
920  @internal
921 */
922 template<typename N>
923 bool find_not_null_ptr(bm::word_t*** arr, N start, N size, N* pos) BMNOEXCEPT
924 {
925  BM_ASSERT(pos);
926 // BM_ASSERT(start < size);
927 //#if defined(BM64_AVX2) || defined(BM64_AVX512)
928 // TODO: optimization for SIMD based next ptr scan
929 #if 0
930  const unsigned unroll_factor = 4;
931  const unsigned len = (size - start);
932  const unsigned len_unr = len - (len % unroll_factor);
933  unsigned k;
934 
935  arr += start;
936  for (k = 0; k < len_unr; k+=unroll_factor)
937  {
938  if (!avx2_test_all_zero_wave(arr+k))
939  {
940  if (arr[k])
941  {
942  *pos = k + start;
943  return true;
944  }
945  if (arr[k+1])
946  {
947  *pos = k + start + 1;
948  return true;
949  }
950  if (arr[k+2])
951  {
952  *pos = k + start + 2;
953  return true;
954  }
955  if (arr[k+3])
956  {
957  *pos = k + start + 3;
958  return true;
959  }
960  }
961  } // for k
962 
963  for (; k < len; ++k)
964  {
965  if (arr[k])
966  {
967  *pos = k + start;
968  return true;
969  }
970  } // for k
971 #else
972  for (; start < size; ++start)
973  {
974  if (arr[start])
975  {
976  *pos = start;
977  return true;
978  }
979  } // for i
980 #endif
981  return false;
982 }
983 
984 
985 
986 
987 //---------------------------------------------------------------------
988 
989 /*!
990  \brief Lexicographical comparison of two words as bit strings (reference)
991  Auxiliary implementation for testing and reference purposes.
992  \param w1 - First word.
993  \param w2 - Second word.
994  \return <0 - less, =0 - equal, >0 - greater.
995 
996  @ingroup bitfunc
997 */
998 template<typename T> int wordcmp0(T w1, T w2)
999 {
1000  while (w1 != w2)
1001  {
1002  int res = (w1 & 1) - (w2 & 1);
1003  if (res != 0) return res;
1004  w1 >>= 1;
1005  w2 >>= 1;
1006  }
1007  return 0;
1008 }
1009 
1010 
1011 /*
1012 template<typename T> int wordcmp(T w1, T w2)
1013 {
1014  T diff = w1 ^ w2;
1015  return diff ? ((w1 & diff & (diff ^ (diff - 1)))? 1 : -1) : 0;
1016 }
1017 */
1018 /*!
1019  \brief Lexicographical comparison of two words as bit strings.
1020  Auxiliary implementation for testing and reference purposes.
1021  \param a - First word.
1022  \param b - Second word.
1023  \return <0 - less, =0 - equal, >0 - greater.
1024 
1025  @ingroup bitfunc
1026 */
1027 template<typename T> int wordcmp(T a, T b)
1028 {
1029  T diff = a ^ b;
1030  return diff? ( (a & diff & -diff)? 1 : -1 ) : 0;
1031 }
1032 
1033 
1034 // Low bit extraction
1035 // x & (x ^ (x-1))
1036 
1037 
1038 // ----------------------------------------------------------------------
1039 
1040 
1041 /*! @brief Returns "true" if all bits in the block are 0
1042  @ingroup bitfunc
1043 */
1044 inline
1046 {
1047 #if defined(VECT_IS_ZERO_BLOCK)
1048  return VECT_IS_ZERO_BLOCK(start);
1049 #else
1050  const bm::wordop_t* BMRESTRICT blk = (bm::wordop_t*) (start);
1051  const bm::wordop_t* BMRESTRICT blk_end = (bm::wordop_t*)(start + bm::set_block_size);
1052  do
1053  {
1054  if (blk[0] | blk[1] | blk[2] | blk[3])
1055  return false;
1056  blk += 4;
1057  } while (blk < blk_end);
1058  return true;
1059 #endif
1060 }
1061 
1062 // ----------------------------------------------------------------------
1063 
1064 /*!
1065  \brief Checks if GAP block is all-zero.
1066  \param buf - GAP buffer pointer.
1067  \returns true if all-zero.
1068 
1069  @ingroup gapfunc
1070 */
1073 {
1074  // (almost) branchless variant:
1075  return (!(*buf & 1u)) & (!(bm::gap_max_bits - 1 - buf[1]));
1076 }
1077 
1078 /*!
1079  \brief Checks if GAP block is all-one.
1080  \param buf - GAP buffer pointer.
1081  \returns true if all-one.
1082  @ingroup gapfunc
1083 */
1086 {
1087  return ((*buf & 1u) && (buf[1] == bm::gap_max_bits - 1));
1088 }
1089 
1090 /*!
1091  \brief Returs GAP block length.
1092  \param buf - GAP buffer pointer.
1093  \returns GAP block length.
1094 
1095  @ingroup gapfunc
1096 */
1099 {
1100  return (bm::gap_word_t)((*buf >> 3) + 1);
1101 }
1102 
1103 
1104 /*!
1105  \brief Returs GAP block capacity
1106  \param buf - GAP buffer pointer
1107  \param glevel_len - pointer on GAP header word
1108  \returns GAP block capacity.
1109 
1110  @ingroup gapfunc
1111 */
1112 template<typename T>
1113 unsigned
1114 gap_capacity(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1115 {
1116  return glevel_len[(*buf >> 1) & 3];
1117 }
1118 
1119 
1120 /*!
1121  \brief Returs GAP block capacity limit.
1122  \param buf - GAP buffer pointer.
1123  \param glevel_len - GAP lengths table (gap_len_table)
1124  \returns GAP block limit.
1125 
1126  @ingroup gapfunc
1127 */
1128 template<typename T>
1129 unsigned
1130 gap_limit(const T* BMRESTRICT buf, const T* BMRESTRICT glevel_len) BMNOEXCEPT
1131 {
1132  return glevel_len[(*buf >> 1) & 3]-4;
1133 }
1134 
1135 
1136 /*!
1137  \brief Returs GAP blocks capacity level.
1138  \param buf - GAP buffer pointer.
1139  \returns GAP block capacity level.
1140 
1141  @ingroup gapfunc
1142 */
1143 template<typename T>
1145 {
1146  return T((*buf >> 1) & 3u);
1147 }
1148 
1149 
1150 /*!
1151  \brief GAP block find the last set bit
1152 
1153  \param buf - GAP buffer pointer.
1154  \param last - index of the last 1 bit
1155 
1156  \return 0 if 1 bit was NOT found
1157 
1158  @ingroup gapfunc
1159 */
1160 template<typename T>
1161 unsigned
1162 gap_find_last(const T* BMRESTRICT buf, unsigned* BMRESTRICT last) BMNOEXCEPT
1163 {
1164  BM_ASSERT(last);
1165 
1166  T is_set = (*buf) & 1u;
1167  T end = T((*buf) >> 3u);
1168 
1169  BM_ASSERT(buf[end] == bm::gap_max_bits - 1);
1170 
1171  is_set ^= T((end-1) & 1u);
1172  if (is_set)
1173  {
1174  *last = buf[end];
1175  return is_set;
1176  }
1177  *last = buf[--end];
1178  return end;
1179 }
1180 
1181 /*!
1182  \brief GAP block find the first set bit
1183 
1184  \param buf - GAP buffer pointer.
1185  \param first - index of the first 1 bit
1186 
1187  \return 0 if 1 bit was NOT found
1188 
1189  @ingroup gapfunc
1190 */
1191 template<typename T>
1192 unsigned
1193 gap_find_first(const T* BMRESTRICT buf, unsigned* BMRESTRICT first) BMNOEXCEPT
1194 {
1195  BM_ASSERT(first);
1196 
1197  T is_set = (*buf) & 1u;
1198  if (is_set)
1199  {
1200  *first = 0;
1201  return is_set;
1202  }
1203  if (buf[1] == bm::gap_max_bits - 1)
1204  return 0;
1205  *first = buf[1] + 1;
1206  return 1;
1207 }
1208 
1209 
1210 
1211 /*
1212  \brief Binary search for the block where bit = pos located.
1213  \param buf - GAP buffer pointer.
1214  \param pos - index of the element.
1215  \param is_set - output. GAP value (0 or 1).
1216  \return GAP index.
1217  @ingroup gapfunc
1218 */
1219 template<typename T>
1220 unsigned gap_bfind(const T* BMRESTRICT buf,
1221  unsigned pos, unsigned* BMRESTRICT is_set) BMNOEXCEPT
1222 {
1223  BM_ASSERT(pos < bm::gap_max_bits);
1224  #undef VECT_GAP_BFIND // TODO: VECTOR bfind causes performance degradation
1225  #ifdef VECT_GAP_BFIND
1226  return VECT_GAP_BFIND(buf, pos, is_set);
1227  #else
1228  *is_set = (*buf) & 1;
1229 
1230  unsigned start = 1;
1231  unsigned end = 1 + ((*buf) >> 3);
1232 
1233  while ( start != end )
1234  {
1235  unsigned curr = (start + end) >> 1;
1236  if ( buf[curr] < pos )
1237  start = curr + 1;
1238  else
1239  end = curr;
1240  }
1241  *is_set ^= ((start-1) & 1);
1242  return start;
1243  #endif
1244 }
1245 
1246 
1247 /*!
1248  \brief Tests if bit = pos is true.
1249  \param buf - GAP buffer pointer.
1250  \param pos - index of the element.
1251  \return true if position is in "1" gap
1252  @ingroup gapfunc
1253 */
1254 template<typename T>
1255 unsigned gap_test(const T* BMRESTRICT buf, unsigned pos) BMNOEXCEPT
1256 {
1257  BM_ASSERT(pos < bm::gap_max_bits);
1258 
1259  unsigned start = 1;
1260  unsigned end = 1 + ((*buf) >> 3);
1261  if (end - start < 10)
1262  {
1263  unsigned sv = *buf & 1;
1264  unsigned sv1= sv ^ 1;
1265  if (buf[1] >= pos) return sv;
1266  if (buf[2] >= pos) return sv1;
1267  if (buf[3] >= pos) return sv;
1268  if (buf[4] >= pos) return sv1;
1269  if (buf[5] >= pos) return sv;
1270  if (buf[6] >= pos) return sv1;
1271  if (buf[7] >= pos) return sv;
1272  if (buf[8] >= pos) return sv1;
1273  if (buf[9] >= pos) return sv;
1274  BM_ASSERT(0);
1275  }
1276  else
1277  {
1278  while (start != end)
1279  {
1280  unsigned curr = (start + end) >> 1;
1281  if (buf[curr] < pos)
1282  start = curr + 1;
1283  else
1284  end = curr;
1285  }
1286  }
1287  return ((*buf) & 1) ^ ((--start) & 1);
1288 }
1289 
1290 /*!
1291  \brief Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
1292  \param buf - GAP buffer pointer.
1293  \param pos - index of the element.
1294  \return true if position is in "1" gap
1295  @ingroup gapfunc
1296 */
1297 template<typename T>
1298 unsigned gap_test_unr(const T* BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
1299 {
1300  BM_ASSERT(pos < bm::gap_max_bits);
1301 
1302  if (pos == 0) // quick answer possible
1303  {
1304  return (*buf) & 1;
1305  }
1306 #if defined(BMSSE2OPT)
1307  unsigned res = bm::sse2_gap_test(buf, pos);
1308  BM_ASSERT(res == bm::gap_test(buf, pos));
1309 #elif defined(BMSSE42OPT)
1310  unsigned res = bm::sse42_gap_test(buf, pos);
1311  BM_ASSERT(res == bm::gap_test(buf, pos));
1312 #elif defined(BMAVX2OPT)
1313  unsigned res = bm::avx2_gap_test(buf, pos);
1314  BM_ASSERT(res == bm::gap_test(buf, pos));
1315 #else
1316  unsigned res = bm::gap_test(buf, pos);
1317 #endif
1318  return res;
1319 }
1320 
1321 /*! For each non-zero block in [from, to] executes supplied functor
1322  \internal
1323 */
1324 template<typename T, typename N, typename F>
1325 void for_each_nzblock_range(T*** root,
1326  N top_size, N nb_from, N nb_to, F& f) BMNOEXCEPT
1327 {
1328  BM_ASSERT(top_size);
1329  if (nb_from > nb_to)
1330  return;
1331  unsigned i_from = unsigned(nb_from >> bm::set_array_shift);
1332  unsigned j_from = unsigned(nb_from & bm::set_array_mask);
1333  unsigned i_to = unsigned(nb_to >> bm::set_array_shift);
1334  unsigned j_to = unsigned(nb_to & bm::set_array_mask);
1335 
1336  if (i_from >= top_size)
1337  return;
1338  if (i_to >= top_size)
1339  {
1340  i_to = unsigned(top_size-1);
1341  j_to = bm::set_sub_array_size-1;
1342  }
1343 
1344  for (unsigned i = i_from; i <= i_to; ++i)
1345  {
1346  T** blk_blk = root[i];
1347  if (!blk_blk)
1348  continue;
1349  if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1350  {
1351  unsigned j = (i == i_from) ? j_from : 0;
1352  if (!j && (i != i_to)) // full sub-block
1353  f.add_full(bm::set_sub_total_bits);
1354  else
1355  {
1356  do
1357  {
1358  f.add_full(bm::gap_max_bits);
1359  if ((i == i_to) && (j == j_to))
1360  return;
1361  } while (++j < bm::set_sub_array_size);
1362  }
1363  }
1364  else
1365  {
1366  unsigned j = (i == i_from) ? j_from : 0;
1367  do
1368  {
1369  if (blk_blk[j])
1370  f(blk_blk[j]);
1371  if ((i == i_to) && (j == j_to))
1372  return;
1373  } while (++j < bm::set_sub_array_size);
1374  }
1375  } // for i
1376 }
1377 
1378 /*! For each non-zero block executes supplied function.
1379  \internal
1380 */
1381 template<class T, class F>
1382 void for_each_nzblock(T*** root, unsigned size1, F& f)
1383 {
1384  typedef typename F::size_type size_type;
1385  for (unsigned i = 0; i < size1; ++i)
1386  {
1387  T** blk_blk = root[i];
1388  if (!blk_blk)
1389  {
1390  f.on_empty_top(i);
1391  continue;
1392  }
1393  f.on_non_empty_top(i);
1394  if ((bm::word_t*)blk_blk == FULL_BLOCK_FAKE_ADDR)
1395  {
1396  size_type r = i * bm::set_sub_array_size;
1397  unsigned j = 0;
1398  do
1399  {
1400  f(FULL_BLOCK_FAKE_ADDR, r + j);
1401  } while (++j < bm::set_sub_array_size);
1402  continue;
1403  }
1404 
1405  unsigned non_empty_top = 0;
1406  size_type r = i * bm::set_sub_array_size;
1407  unsigned j = 0;
1408  do
1409  {
1410 #if defined(BM64_AVX2) || defined(BM64_AVX512)
1411  if (!avx2_test_all_zero_wave(blk_blk + j))
1412  {
1413  non_empty_top = 1;
1414  T* blk0 = blk_blk[j + 0];
1415  T* blk1 = blk_blk[j + 1];
1416  T* blk2 = blk_blk[j + 2];
1417  T* blk3 = blk_blk[j + 3];
1418 
1419  size_type block_idx = r + j + 0;
1420  if (blk0)
1421  f(blk0, block_idx);
1422  else
1423  f.on_empty_block(block_idx);
1424 
1425  if (blk1)
1426  f(blk1, block_idx + 1);
1427  else
1428  f.on_empty_block(block_idx + 1);
1429 
1430  if (blk2)
1431  f(blk2, block_idx + 2);
1432  else
1433  f.on_empty_block(block_idx + 2);
1434 
1435  if (blk3)
1436  f(blk3, block_idx + 3);
1437  else
1438  f.on_empty_block(block_idx + 3);
1439  }
1440  else
1441  {
1442  f.on_empty_block(r + j + 0); f.on_empty_block(r + j + 1);
1443  f.on_empty_block(r + j + 2); f.on_empty_block(r + j + 3);
1444  }
1445  j += 4;
1446 #elif defined(BM64_SSE4)
1447  if (!sse42_test_all_zero_wave((blk_blk + j)))
1448  {
1449  non_empty_top = 1;
1450  T* blk0 = blk_blk[j + 0];
1451  T* blk1 = blk_blk[j + 1];
1452 
1453  size_type block_idx = r + j + 0;
1454  if (blk0)
1455  f(blk0, block_idx);
1456  else
1457  f.on_empty_block(block_idx);
1458 
1459  ++block_idx;
1460  if (blk1)
1461  f(blk1, block_idx);
1462  else
1463  f.on_empty_block(block_idx);
1464  }
1465  else
1466  {
1467  f.on_empty_block(r + j + 0);
1468  f.on_empty_block(r + j + 1);
1469  }
1470  j += 2;
1471 #else
1472  if (blk_blk[j])
1473  {
1474  f(blk_blk[j], r + j);
1475  non_empty_top = 1;
1476  }
1477  else
1478  f.on_empty_block(r + j);
1479  ++j;
1480 #endif
1481  } while (j < bm::set_sub_array_size);
1482 
1483  if (non_empty_top == 0)
1484  f.on_empty_top(i);
1485  } // for i
1486 }
1487 
1488 /*! For each non-zero block executes supplied function.
1489 */
1490 template<class T, class F>
1491 void for_each_nzblock2(T*** root, unsigned size1, F& f)
1492 {
1493 #ifdef BM64_SSE4
1494  for (unsigned i = 0; i < size1; ++i)
1495  {
1496  T** blk_blk;
1497  if ((blk_blk = root[i])!=0)
1498  {
1499  if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1500  {
1501  for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1502  {
1504  }
1505  continue;
1506  }
1507  {
1508  __m128i w0;
1509  for (unsigned j = 0; j < bm::set_sub_array_size; j+=4)
1510  {
1511  w0 = _mm_loadu_si128((__m128i*)(blk_blk + j));
1512  if (!_mm_testz_si128(w0, w0))
1513  {
1514  T* blk0 = blk_blk[j + 0];
1515  T* blk1 = blk_blk[j + 1];
1516 
1517  if (blk0)
1518  f(blk0);
1519  if (blk1)
1520  f(blk1);
1521  }
1522  w0 = _mm_loadu_si128((__m128i*)(blk_blk + j + 2));
1523  if (!_mm_testz_si128(w0, w0))
1524  {
1525  T* blk0 = blk_blk[j + 2];
1526  T* blk1 = blk_blk[j + 3];
1527  if (blk0)
1528  f(blk0);
1529  if (blk1)
1530  f(blk1);
1531  }
1532  } // for j
1533  }
1534  }
1535  } // for i
1536 #elif defined(BM64_AVX2) || defined(BM64_AVX512)
1537  for (unsigned i = 0; i < size1; ++i)
1538  {
1539  T** blk_blk;
1540  if ((blk_blk = root[i]) != 0)
1541  {
1542  if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1543  {
1544  for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1545  {
1547  }
1548  continue;
1549  }
1550  {
1551  for (unsigned j = 0; j < bm::set_sub_array_size; j += 4)
1552  {
1553  __m256i w0 = _mm256_loadu_si256((__m256i*)(blk_blk + j));
1554  if (!_mm256_testz_si256(w0, w0))
1555  {
1556  // as a variant could use: blk0 = (T*)_mm256_extract_epi64(w0, 0);
1557  // but it measures marginally slower
1558  T* blk0 = blk_blk[j + 0];
1559  T* blk1 = blk_blk[j + 1];
1560  T* blk2 = blk_blk[j + 2];
1561  T* blk3 = blk_blk[j + 3];
1562  if (blk0)
1563  f(blk0);
1564  if (blk1)
1565  f(blk1);
1566  if (blk2)
1567  f(blk2);
1568  if (blk3)
1569  f(blk3);
1570  }
1571  } // for j
1572  }
1573  }
1574  } // for i
1575 
1576 #else // non-SIMD mode
1577  for (unsigned i = 0; i < size1; ++i)
1578  {
1579  T** blk_blk;
1580  if ((blk_blk = root[i])!=0)
1581  {
1582  if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1583  {
1584  for (unsigned j = 0; j < bm::set_sub_array_size; ++j)
1585  {
1587  }
1588  continue;
1589  }
1590  unsigned j = 0;
1591  do
1592  {
1593  if (blk_blk[j])
1594  f(blk_blk[j]);
1595  if (blk_blk[j+1])
1596  f(blk_blk[j+1]);
1597  if (blk_blk[j+2])
1598  f(blk_blk[j+2]);
1599  if (blk_blk[j+3])
1600  f(blk_blk[j+3]);
1601  j += 4;
1602  } while (j < bm::set_sub_array_size);
1603  }
1604  } // for i
1605 #endif
1606 }
1607 
1608 
1609 /*! For each non-zero block executes supplied function-predicate.
1610  Function returns if function-predicate returns true
1611 */
1612 template<typename T, typename BI, typename F>
1613 bool for_each_nzblock_if(T*** root, BI size1, F& f) BMNOEXCEPT
1614 {
1615  BI block_idx = 0;
1616  for (BI i = 0; i < size1; ++i)
1617  {
1618  T** blk_blk = root[i];
1619  if (!blk_blk)
1620  {
1621  block_idx += bm::set_sub_array_size;
1622  continue;
1623  }
1624  if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1625  {
1626  for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1627  {
1628  if (f(FULL_BLOCK_FAKE_ADDR, block_idx))
1629  return true;
1630  } // for j
1631  continue;
1632  }
1633 
1634  for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1635  {
1636  if (blk_blk[j])
1637  if (f(blk_blk[j], block_idx))
1638  return true;
1639  } // for j
1640  } // for i
1641  return false;
1642 }
1643 
1644 /*! For each block executes supplied function.
1645 */
1646 template<class T, class F, typename BLOCK_IDX>
1647 void for_each_block(T*** root, unsigned size1, F& f, BLOCK_IDX start)
1648 {
1649  BLOCK_IDX block_idx = start;
1650  for (unsigned i = 0; i < size1; ++i)
1651  {
1652  T** blk_blk = root[i];
1653  if (blk_blk)
1654  {
1655  if (blk_blk == (T**)FULL_BLOCK_FAKE_ADDR)
1656  {
1657  for (unsigned j = 0; j < bm::set_sub_array_size; ++j, ++block_idx)
1658  {
1659  f(FULL_BLOCK_FAKE_ADDR, block_idx);
1660  }
1661  continue;
1662  }
1663  for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1664  {
1665  f(blk_blk[j], block_idx);
1666  }
1667  }
1668  else
1669  {
1670  for (unsigned j = 0;j < bm::set_sub_array_size; ++j, ++block_idx)
1671  {
1672  f(0, block_idx);
1673  }
1674  }
1675  }
1676 }
1677 
1678 
1679 
1680 /*! Special BM optimized analog of STL for_each
1681 */
1682 template<class T, class F> F bmfor_each(T first, T last, F f)
1683 {
1684  do
1685  {
1686  f(*first);
1687  ++first;
1688  } while (first < last);
1689  return f;
1690 }
1691 
1692 /*! Computes SUM of all elements of the sequence
1693 */
1694 template<typename T>
1695 bm::id64_t sum_arr(const T* first, const T* last) BMNOEXCEPT
1696 {
1697  bm::id64_t sum = 0;
1698  for (;first < last; ++first)
1699  sum += *first;
1700  return sum;
1701 }
1702 
1703 /*!
1704  Extract short (len=1) exceptions from the GAP block
1705  \param buf - GAP buffer to split
1706  \param arr0 - [OUT] list of isolates 0 positions (clear list)
1707  \param arr1 - [OUT] list of isolated 1 positions (set list)
1708  \param arr0_cnt - [OUT] arr0 size
1709  \param arr1_cnt -
1710  @ingroup gapfunc
1711 */
1712 template<typename T>
1713 void gap_split(const T* buf,
1714  T* arr0, T* arr1, T& arr0_cnt, T& arr1_cnt) BMNOEXCEPT
1715 {
1716  const T* pcurr = buf;
1717  unsigned len = (*pcurr >> 3);
1718  const T* pend = pcurr + len;
1719 
1720  T cnt0, cnt1;
1721  cnt0 = cnt1 = 0;
1722  unsigned is_set = (*buf & 1);
1723 
1724  if (*pcurr == 0)
1725  {
1726  if (is_set)
1727  {
1728  arr1[cnt1] = *pcurr;
1729  ++cnt1;
1730  }
1731  else
1732  {
1733  arr0[cnt0] = *pcurr;
1734  ++cnt0;
1735  }
1736  }
1737  T prev = *pcurr;
1738  ++pcurr;
1739 
1740  while (pcurr <= pend)
1741  {
1742  is_set ^= 1;
1743  T delta = *pcurr - prev;
1744  if (delta == 1)
1745  {
1746  if (is_set)
1747  {
1748  arr1[cnt1] = prev;
1749  ++cnt1;
1750  }
1751  else
1752  {
1753  arr0[cnt0] = prev;
1754  ++cnt0;
1755  }
1756  }
1757  prev = *pcurr++;
1758  } // while
1759 
1760  arr0_cnt = cnt0;
1761  arr1_cnt = cnt1;
1762 }
1763 
1764 
1765 /*!
1766  \brief Calculates number of bits ON in GAP buffer.
1767  \param buf - GAP buffer pointer.
1768  \param dsize - buffer size
1769  \return Number of non-zero bits.
1770  @ingroup gapfunc
1771 */
1772 template<typename T>
1773 unsigned gap_bit_count(const T* buf, unsigned dsize=0) BMNOEXCEPT
1774 {
1775  const T* pcurr = buf;
1776  if (dsize == 0)
1777  dsize = (*pcurr >> 3);
1778 
1779  const T* pend = pcurr + dsize;
1780 
1781  unsigned bits_counter = 0;
1782  ++pcurr;
1783 
1784  if (*buf & 1)
1785  {
1786  bits_counter += *pcurr + 1;
1787  ++pcurr;
1788  }
1789  for (++pcurr; pcurr <= pend; pcurr += 2)
1790  bits_counter += *pcurr - *(pcurr-1);
1791  return bits_counter;
1792 }
1793 
1794 /*!
1795  \brief Calculates number of bits ON in GAP buffer. Loop unrolled version.
1796  \param buf - GAP buffer pointer.
1797  \return Number of non-zero bits.
1798  @ingroup gapfunc
1799 */
1800 template<typename T>
1801 unsigned gap_bit_count_unr(const T* buf) BMNOEXCEPT
1802 {
1803  const T* pcurr = buf;
1804  unsigned dsize = (*pcurr >> 3);
1805 
1806  unsigned cnt = 0;
1807  pcurr = buf + 1; // set up start position
1808  T first_one = *buf & 1;
1809  if (first_one)
1810  {
1811  cnt += *pcurr + 1;
1812  ++pcurr;
1813  }
1814  ++pcurr; // set GAP to 1
1815 
1816  #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
1817  if (dsize > 34)
1818  {
1819  const unsigned unr_factor = 32;
1820  unsigned waves = (dsize-2) / unr_factor;
1821  pcurr = avx2_gap_sum_arr(pcurr, waves, &cnt);
1822  }
1823  #elif defined(BMSSE42OPT) || defined(BMSSE2OPT)
1824  if (dsize > 18)
1825  {
1826  const unsigned unr_factor = 16;
1827  unsigned waves = (dsize - 2) / unr_factor;
1828  pcurr = sse2_gap_sum_arr(pcurr, waves, &cnt);
1829  }
1830  #else
1831  if (dsize > 10)
1832  {
1833  const unsigned unr_factor = 8;
1834  unsigned waves = (dsize - 2) / unr_factor;
1835  for (unsigned i = 0; i < waves; i += unr_factor)
1836  {
1837  cnt += pcurr[0] - pcurr[0 - 1];
1838  cnt += pcurr[2] - pcurr[2 - 1];
1839  cnt += pcurr[4] - pcurr[4 - 1];
1840  cnt += pcurr[6] - pcurr[6 - 1];
1841 
1842  pcurr += unr_factor;
1843  } // for
1844  }
1845  #endif
1846 
1847  const T* pend = buf + dsize;
1848  for ( ; pcurr <= pend ; pcurr+=2)
1849  {
1850  cnt += *pcurr - *(pcurr - 1);
1851  }
1852  BM_ASSERT(cnt == bm::gap_bit_count(buf));
1853  return cnt;
1854 }
1855 
1856 
1857 
1858 /*!
1859  \brief Counts 1 bits in GAP buffer in the closed [left, right] range.
1860  \param buf - GAP buffer pointer.
1861  \param left - leftmost bit index to start from
1862  \param right- rightmost bit index
1863  \return Number of non-zero bits.
1864  @ingroup gapfunc
1865 */
1866 template<typename T>
1867 unsigned gap_bit_count_range(const T* const buf,
1868  unsigned left, unsigned right) BMNOEXCEPT
1869 {
1870  BM_ASSERT(left <= right);
1871  BM_ASSERT(right < bm::gap_max_bits);
1872 
1873  const T* pcurr = buf;
1874  const T* pend = pcurr + (*pcurr >> 3);
1875 
1876  unsigned bits_counter = 0;
1877  unsigned is_set;
1878  unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1879  is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
1880 
1881  pcurr = buf + start_pos;
1882  if (right <= *pcurr) // we are in the target gap right now
1883  {
1884  bits_counter = unsigned(right - left + 1u) & is_set; // & is_set == if(is_set)
1885  return bits_counter;
1886  }
1887  bits_counter += unsigned(*pcurr - left + 1u) & is_set;
1888 
1889  unsigned prev_gap = *pcurr++;
1890  for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
1891  {
1892  bits_counter += (*pcurr - prev_gap) & is_set;
1893  if (pcurr == pend)
1894  return bits_counter;
1895  prev_gap = *pcurr++;
1896  }
1897  bits_counter += unsigned(right - prev_gap) & is_set;
1898  return bits_counter;
1899 }
1900 
1901 /*!
1902  \brief Test if all bits are 1 in GAP buffer in the [left, right] range.
1903  \param buf - GAP buffer pointer.
1904  \param left - leftmost bit index to start from
1905  \param right- rightmost bit index
1906  \return true if all bits are "11111"
1907  @ingroup gapfunc
1908 */
1909 template<typename T>
1910 bool gap_is_all_one_range(const T* const BMRESTRICT buf,
1911  unsigned left, unsigned right) BMNOEXCEPT
1912 {
1913  BM_ASSERT(left <= right);
1914  BM_ASSERT(right < bm::gap_max_bits);
1915 
1916  unsigned is_set;
1917  unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1918  if (!is_set) // GAP is 0
1919  return false;
1920  const T* const pcurr = buf + start_pos;
1921  return (right <= *pcurr);
1922 }
1923 
1924 /*!
1925  \brief Test if any bits are 1 in GAP buffer in the [left, right] range.
1926  \param buf - GAP buffer pointer.
1927  \param left - leftmost bit index to start from
1928  \param right- rightmost bit index
1929  \return true if at least 1 "00010"
1930  @ingroup gapfunc
1931 */
1932 template<typename T>
1933 bool gap_any_range(const T* const BMRESTRICT buf,
1934  unsigned left, unsigned right) BMNOEXCEPT
1935 {
1936  BM_ASSERT(left <= right);
1937  BM_ASSERT(right < bm::gap_max_bits);
1938 
1939  unsigned is_set;
1940  unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1941  const T* const pcurr = buf + start_pos;
1942 
1943  if (!is_set) // start GAP is 0 ...
1944  {
1945  if (right <= *pcurr) // ...bit if the interval goes into at least 1 blk
1946  return false; // .. nope
1947  return true;
1948  }
1949  return true;
1950 }
1951 
1952 /*!
1953  \brief Test if any bits are 1 in GAP buffer in the [left, right] range
1954  and flanked with 0s
1955  \param buf - GAP buffer pointer.
1956  \param left - leftmost bit index to start from
1957  \param right- rightmost bit index
1958  \return true if "011110"
1959  @ingroup gapfunc
1960 */
1961 template<typename T>
1962 bool gap_is_interval(const T* const BMRESTRICT buf,
1963  unsigned left, unsigned right) BMNOEXCEPT
1964 {
1965  BM_ASSERT(left <= right);
1966  BM_ASSERT(left > 0); // cannot check left-1 otherwise
1967  BM_ASSERT(right < bm::gap_max_bits-1); // cannot check right+1 otherwise
1968 
1969  unsigned is_set;
1970  unsigned start_pos = bm::gap_bfind(buf, left, &is_set);
1971 
1972  const T* pcurr = buf + start_pos;
1973  if (!is_set || (right != *pcurr) || (start_pos <= 1))
1974  return false;
1975  --pcurr;
1976  if (*pcurr != left-1)
1977  return false;
1978  return true;
1979 }
1980 
1981 /**
1982  \brief Searches for the last 1 bit in the 111 interval of a GAP block
1983  \param buf - BIT block buffer
1984  \param nbit - bit index to start checking from
1985  \param pos - [out] found value
1986 
1987  \return false if not found
1988  @ingroup gapfunc
1989 */
1990 template<typename T>
1991 bool gap_find_interval_end(const T* const BMRESTRICT buf,
1992  unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
1993 {
1994  BM_ASSERT(pos);
1995  BM_ASSERT(nbit < bm::gap_max_bits);
1996 
1997  unsigned is_set;
1998  unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
1999  if (!is_set)
2000  return false;
2001  *pos = buf[start_pos];
2002  return true;
2003 }
2004 
2005 
2006 /**
2007  \brief Searches for the first 1 bit in the 111 interval of a GAP block
2008  \param buf - BIT block buffer
2009  \param nbit - bit index to start checking from
2010  \param pos - [out] found value
2011 
2012  \return false if not found
2013  @ingroup gapfunc
2014 */
2015 template<typename T>
2016 bool gap_find_interval_start(const T* const BMRESTRICT buf,
2017  unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
2018 {
2019  BM_ASSERT(pos);
2020  BM_ASSERT(nbit < bm::gap_max_bits);
2021 
2022  unsigned is_set;
2023  unsigned start_pos = bm::gap_bfind(buf, nbit, &is_set);
2024  if (!is_set)
2025  return false;
2026  --start_pos;
2027  if (!start_pos)
2028  *pos = 0;
2029  else
2030  *pos = buf[start_pos]+1;
2031  return true;
2032 }
2033 
2034 
2035 /*!
2036  \brief GAP block find position for the rank
2037 
2038  \param block - bit block buffer pointer
2039  \param rank - rank to find (must be > 0)
2040  \param nbit_from - start bit position in block
2041  \param nbit_pos - found position
2042 
2043  \return 0 if position with rank was found, or
2044  the remaining rank (rank - population count)
2045 
2046  @ingroup gapfunc
2047 */
2048 template<typename T, typename SIZE_TYPE>
2049 SIZE_TYPE gap_find_rank(const T* const block,
2050  SIZE_TYPE rank,
2051  unsigned nbit_from,
2052  unsigned& nbit_pos) BMNOEXCEPT
2053 {
2054  BM_ASSERT(block);
2055  BM_ASSERT(rank);
2056 
2057  const T* pcurr = block;
2058  const T* pend = pcurr + (*pcurr >> 3);
2059 
2060  unsigned bits_counter = 0;
2061  unsigned is_set;
2062  unsigned start_pos = bm::gap_bfind(block, nbit_from, &is_set);
2063  is_set = ~(is_set - 1u); // 0xFFF.. if true (mask for branchless code)
2064 
2065  pcurr = block + start_pos;
2066  bits_counter += unsigned(*pcurr - nbit_from + 1u) & is_set;
2067  if (bits_counter >= rank) // found!
2068  {
2069  nbit_pos = nbit_from + unsigned(rank) - 1u;
2070  return 0;
2071  }
2072  rank -= bits_counter;
2073  unsigned prev_gap = *pcurr++;
2074  for (is_set ^= ~0u; pcurr <= pend; is_set ^= ~0u)
2075  {
2076  bits_counter = (*pcurr - prev_gap) & is_set;
2077  if (bits_counter >= rank) // found!
2078  {
2079  nbit_pos = prev_gap + unsigned(rank);
2080  return 0;
2081  }
2082  rank -= bits_counter;
2083  prev_gap = *pcurr++;
2084  } // for
2085 
2086  return rank;
2087 }
2088 
2089 
2090 
2091 /*!
2092  \brief Counts 1 bits in GAP buffer in the closed [0, right] range.
2093  \param buf - GAP buffer pointer.
2094  \param right- rightmost bit index
2095  \param is_corrected - if true the result will be rank corrected
2096  if right bit == true count=count-1
2097  \return Number of non-zero bits
2098  @ingroup gapfunc
2099 */
2100 template<typename T>
2101 unsigned gap_bit_count_to(const T* const buf, T right,
2102  bool is_corrected=false) BMNOEXCEPT
2103 {
2104  const T* pcurr = buf;
2105  const T* pend = pcurr + (*pcurr >> 3);
2106 
2107  unsigned bits_counter = 0;
2108  unsigned is_set = ~((unsigned(*buf) & 1u) - 1u); // 0xFFF.. if true (mask for branchless code)
2109  BM_ASSERT(is_set == 0u || is_set == ~0u);
2110  pcurr = buf + 1;
2111 
2112  if (right <= *pcurr) // we are in the target block right now
2113  {
2114  bits_counter = (right + 1u) & is_set; // & is_set == if (is_set)
2115  bits_counter -= (is_set & is_corrected);
2116  return bits_counter;
2117  }
2118  bits_counter += (*pcurr + 1u) & is_set;
2119 
2120  unsigned prev_gap = *pcurr++;
2121  for (is_set ^= ~0u; right > *pcurr; is_set ^= ~0u)
2122  {
2123  bits_counter += (*pcurr - prev_gap) & is_set;
2124  if (pcurr == pend)
2125  {
2126  bits_counter -= (is_set & is_corrected);
2127  return bits_counter;
2128  }
2129  prev_gap = *pcurr++;
2130  }
2131  bits_counter += (right - prev_gap) & is_set;
2132  bits_counter -= (is_set & is_corrected);
2133  return bits_counter;
2134 }
2135 
2136 
2137 /*!
2138  D-GAP block for_each algorithm
2139 
2140  D-Gap Functor is called for each element but last one.
2141 
2142  \param gap_buf - GAP buffer
2143  \param func - functor object
2144 
2145 */
2146 template<class T, class Func>
2147 void for_each_dgap(const T* gap_buf, Func& func)
2148 {
2149  const T* pcurr = gap_buf;
2150  const T* pend = pcurr + (*pcurr >> 3);
2151  ++pcurr;
2152 
2153  T prev = *pcurr;
2154  func((T)(prev + 1)); // first element incremented to avoid 0
2155  ++pcurr;
2156  do
2157  {
2158  func((T)(*pcurr - prev)); // all others are [N] - [N-1]
2159  prev = *pcurr;
2160  } while (++pcurr < pend);
2161 }
2162 
2163 /** d-Gap copy functor
2164  @internal
2165 */
2166 template<typename T> struct d_copy_func
2167 {
2168  d_copy_func(T* dg_buf) : dgap_buf_(dg_buf) {}
2169  void operator()(T dgap) { *dgap_buf_++ = dgap; }
2170 
2172 };
2173 
2174 /*!
2175  \brief Convert GAP buffer into D-GAP buffer
2176 
2177  Delta GAP representation is DGAP[N] = GAP[N] - GAP[N-1]
2178 
2179  \param gap_buf - GAP buffer
2180  \param dgap_buf - Delta-GAP buffer
2181  \param copy_head - flag to copy GAP header
2182 
2183  \internal
2184 
2185  @ingroup gapfunc
2186 */
2187 template<typename T>
2188 T* gap_2_dgap(const T* BMRESTRICT gap_buf,
2189  T* BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
2190 {
2191  if (copy_head) // copy GAP header
2192  {
2193  *dgap_buf++ = *gap_buf;
2194  }
2195 
2196  d_copy_func<T> copy_func(dgap_buf);
2197  for_each_dgap<T, d_copy_func<T> >(gap_buf, copy_func);
2198  return copy_func.dgap_buf_;
2199 }
2200 
2201 /*!
2202  \brief Convert D-GAP buffer into GAP buffer
2203 
2204  GAP representation is GAP[N] = DGAP[N] + DGAP[N-1]
2205 
2206  \param dgap_buf - Delta-GAP buffer
2207  \param gap_header - GAP header word
2208  \param gap_buf - GAP buffer
2209 
2210  \internal
2211  @ingroup gapfunc
2212 */
2213 template<typename T>
2214 void dgap_2_gap(const T* BMRESTRICT dgap_buf,
2215  T* BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
2216 {
2217  const T* pcurr = dgap_buf;
2218  unsigned len;
2219  if (!gap_header) // GAP header is already part of the stream
2220  {
2221  len = *pcurr >> 3;
2222  *gap_buf++ = *pcurr++; // copy GAP header
2223  }
2224  else // GAP header passed as a parameter
2225  {
2226  len = gap_header >> 3;
2227  *gap_buf++ = gap_header; // assign GAP header
2228  }
2229  --len; // last element is actually not encoded
2230  const T* pend = pcurr + len;
2231 
2232  *gap_buf = *pcurr++; // copy first element
2233  if (*gap_buf == 0)
2234  *gap_buf = 65535; // fix +1 overflow
2235  else
2236  *gap_buf = T(*gap_buf - 1);
2237 
2238  for (++gap_buf; pcurr < pend; ++pcurr)
2239  {
2240  T prev = *(gap_buf-1); // don't remove temp(undef expression!)
2241  *gap_buf++ = T(*pcurr + prev);
2242  }
2243  *gap_buf = 65535; // add missing last element
2244 }
2245 
2246 
2247 /*!
2248  \brief Lexicographical comparison of GAP buffers.
2249  \param buf1 - First GAP buffer pointer.
2250  \param buf2 - Second GAP buffer pointer.
2251  \return <0 - less, =0 - equal, >0 - greater.
2252 
2253  @ingroup gapfunc
2254 */
2255 template<typename T>
2256 int gapcmp(const T* buf1, const T* buf2) BMNOEXCEPT
2257 {
2258  const T* pcurr1 = buf1;
2259  const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2260  unsigned bitval1 = *buf1 & 1;
2261  ++pcurr1;
2262 
2263  const T* pcurr2 = buf2;
2264  unsigned bitval2 = *buf2 & 1;
2265  ++pcurr2;
2266 
2267  while (pcurr1 <= pend1)
2268  {
2269  if (*pcurr1 == *pcurr2)
2270  {
2271  if (bitval1 != bitval2)
2272  {
2273  return (bitval1) ? 1 : -1;
2274  }
2275  }
2276  else
2277  {
2278  if (bitval1 == bitval2)
2279  {
2280  if (bitval1)
2281  {
2282  return (*pcurr1 < *pcurr2) ? -1 : 1;
2283  }
2284  else
2285  {
2286  return (*pcurr1 < *pcurr2) ? 1 : -1;
2287  }
2288  }
2289  else
2290  {
2291  return (bitval1) ? 1 : -1;
2292  }
2293  }
2294 
2295  ++pcurr1; ++pcurr2;
2296 
2297  bitval1 ^= 1;
2298  bitval2 ^= 1;
2299  }
2300 
2301  return 0;
2302 }
2303 
2304 /*!
2305  \brief Find first bit which is different between two GAP-blocks
2306  \param buf1 - block 1
2307  \param buf2 - block 2
2308  \param pos - out - position of difference (undefined if blocks are equal)
2309  \return true if difference was found
2310 
2311  @ingroup gapfunc
2312 */
2313 template<typename T>
2314 bool gap_find_first_diff(const T* BMRESTRICT buf1,
2315  const T* BMRESTRICT buf2,
2316  unsigned* BMRESTRICT pos) BMNOEXCEPT
2317 {
2318  BM_ASSERT(buf1 && buf2 && pos);
2319 
2320  const T* pcurr1 = buf1;
2321  const T* pend1 = pcurr1 + (*pcurr1 >> 3);
2322  const T* pcurr2 = buf2;
2323  for (++pcurr1, ++pcurr2; pcurr1 <= pend1; ++pcurr1, ++pcurr2)
2324  {
2325  if (*pcurr1 != *pcurr2)
2326  {
2327  *pos = 1 + ((*pcurr1 < *pcurr2) ? *pcurr1 : *pcurr2);
2328  return true;
2329  }
2330  } // for
2331  return false;
2332 }
2333 
2334 // -------------------------------------------------------------------------
2335 //
2336 
2337 /*!
2338  \brief Abstract operation for GAP buffers.
2339  Receives functor F as a template argument
2340  \param dest - destination memory buffer.
2341  \param vect1 - operand 1 GAP encoded buffer.
2342  \param vect1_mask - XOR mask for starting bitflag for vector1
2343  can be 0 or 1 (1 inverts the vector)
2344  \param vect2 - operand 2 GAP encoded buffer.
2345  \param vect2_mask - same as vect1_mask
2346  \param f - operation functor.
2347  \param dlen - destination length after the operation
2348 
2349  \note Internal function.
2350  @internal
2351 
2352  @ingroup gapfunc
2353 */
2354 template<typename T, class F>
2355 void gap_buff_op(T* BMRESTRICT dest,
2356  const T* BMRESTRICT vect1,
2357  unsigned vect1_mask,
2358  const T* BMRESTRICT vect2,
2359  unsigned vect2_mask,
2360  F& f,
2361  unsigned& dlen) BMNOEXCEPT2
2362 {
2363  const T* cur1 = vect1;
2364  const T* cur2 = vect2;
2365 
2366  T bitval1 = (T)((*cur1++ & 1) ^ vect1_mask);
2367  T bitval2 = (T)((*cur2++ & 1) ^ vect2_mask);
2368 
2369  T bitval = (T) f(bitval1, bitval2);
2370  T bitval_prev = bitval;
2371 
2372  T* res = dest;
2373  *res = bitval;
2374  ++res;
2375 
2376  T c1 = *cur1; T c2 = *cur2;
2377  while (1)
2378  {
2379  bitval = (T) f(bitval1, bitval2);
2380 
2381  // Check if GAP value changes and we need to
2382  // start the next one
2383  //
2384  res += (bitval != bitval_prev);
2385  bitval_prev = bitval;
2386  if (c1 < c2) // (*cur1 < *cur2)
2387  {
2388  *res = c1;
2389  ++cur1; c1 = *cur1;
2390  bitval1 ^= 1;
2391  }
2392  else // >=
2393  {
2394  *res = c2;
2395  if (c2 < c1) // (*cur2 < *cur1)
2396  {
2397  bitval2 ^= 1;
2398  }
2399  else // equal
2400  {
2401  if (c2 == (bm::gap_max_bits - 1))
2402  break;
2403 
2404  ++cur1; c1 = *cur1;
2405  bitval1 ^= 1; bitval2 ^= 1;
2406  }
2407  ++cur2; c2 = *cur2;
2408  }
2409  } // while
2410 
2411  dlen = (unsigned)(res - dest);
2412  *dest = (T)((*dest & 7) + (dlen << 3));
2413 }
2414 
2415 
2416 /*!
2417  \brief Abstract operation for GAP buffers (predicts legth)
2418  Receives functor F as a template argument
2419  \param vect1 - operand 1 GAP encoded buffer.
2420  \param vect2 - operand 2 GAP encoded buffer.
2421  \param f - operation functor.
2422  \param dlen - destination length after the operation
2423  \param limit - maximum target length limit,
2424  returns false if limit is reached
2425  \return true if operation would be successfull or
2426  false if limit reached
2427 
2428  \note Internal function.
2429  @internal
2430 
2431  @ingroup gapfunc
2432 */
2433 template<typename T, class F>
2434 bool gap_buff_dry_op(const T* BMRESTRICT vect1,
2435  const T* BMRESTRICT vect2,
2436  F& f,
2437  unsigned& dlen,
2438  unsigned limit) BMNOEXCEPT2
2439 {
2440  const T* cur1 = vect1;
2441  const T* cur2 = vect2;
2442 
2443  T bitval1 = (T)((*cur1++ & 1));
2444  T bitval2 = (T)((*cur2++ & 1));
2445 
2446  T bitval = (T) f(bitval1, bitval2);
2447  T bitval_prev = bitval;
2448 
2449  unsigned len = 1;
2450 
2451  T c1 = *cur1; T c2 = *cur2;
2452  while (1)
2453  {
2454  bitval = (T) f(bitval1, bitval2);
2455 
2456  // Check if GAP value changes and we need to
2457  // start the next one
2458  //
2459  len += (bitval != bitval_prev);
2460  if (len > limit)
2461  return false;
2462  bitval_prev = bitval;
2463  if (c1 < c2)
2464  {
2465  ++cur1; c1 = *cur1;
2466  bitval1 ^= 1;
2467  }
2468  else // >=
2469  {
2470  if (c2 < c1) // (*cur2 < *cur1)
2471  {
2472  bitval2 ^= 1;
2473  }
2474  else // equal
2475  {
2476  if (c2 == (bm::gap_max_bits - 1))
2477  break;
2478 
2479  ++cur1; c1 = *cur1;
2480  bitval1 ^= 1; bitval2 ^= 1;
2481  }
2482  ++cur2; c2 = *cur2;
2483  }
2484 
2485  } // while
2486 
2487  dlen = len;
2488  return true;
2489 }
2490 
2491 
2492 /*!
2493  \brief Abstract distance test operation for GAP buffers.
2494  Receives functor F as a template argument
2495  \param vect1 - operand 1 GAP encoded buffer.
2496  \param vect1_mask - XOR mask for starting bitflag for vector1
2497  can be 0 or 1 (1 inverts the vector)
2498  \param vect2 - operand 2 GAP encoded buffer.
2499  \param vect2_mask - same as vect1_mask
2500  \param f - operation functor.
2501  \note Internal function.
2502  \return non zero value if operation result returns any 1 bit
2503 
2504  @ingroup gapfunc
2505 */
2506 template<typename T, class F>
2507 unsigned gap_buff_any_op(const T* BMRESTRICT vect1,
2508  unsigned vect1_mask,
2509  const T* BMRESTRICT vect2,
2510  unsigned vect2_mask,
2511  F f) BMNOEXCEPT2
2512 {
2513  const T* cur1 = vect1;
2514  const T* cur2 = vect2;
2515 
2516  unsigned bitval1 = (*cur1++ & 1) ^ vect1_mask;
2517  unsigned bitval2 = (*cur2++ & 1) ^ vect2_mask;
2518 
2519  unsigned bitval = f(bitval1, bitval2);
2520  if (bitval)
2521  return bitval;
2522  unsigned bitval_prev = bitval;
2523 
2524  while (1)
2525  {
2526  bitval = f(bitval1, bitval2);
2527  if (bitval)
2528  return bitval;
2529 
2530  if (bitval != bitval_prev)
2531  bitval_prev = bitval;
2532 
2533  if (*cur1 < *cur2)
2534  {
2535  ++cur1;
2536  bitval1 ^= 1;
2537  }
2538  else // >=
2539  {
2540  if (*cur2 < *cur1)
2541  {
2542  bitval2 ^= 1;
2543  }
2544  else // equal
2545  {
2546  if (*cur2 == (bm::gap_max_bits - 1))
2547  {
2548  break;
2549  }
2550 
2551  ++cur1;
2552  bitval1 ^= 1;
2553  bitval2 ^= 1;
2554  }
2555  ++cur2;
2556  }
2557 
2558  } // while
2559 
2560  return 0;
2561 }
2562 
2563 
2564 
2565 /*!
2566  \brief Abstract distance(similarity) operation for GAP buffers.
2567  Receives functor F as a template argument
2568  \param vect1 - operand 1 GAP encoded buffer.
2569  \param vect2 - operand 2 GAP encoded buffer.
2570  \param f - operation functor.
2571  \note Internal function.
2572 
2573  @ingroup gapfunc
2574 */
2575 template<typename T, class F>
2576 unsigned gap_buff_count_op(const T* vect1, const T* vect2, F f) BMNOEXCEPT2
2577 {
2578  unsigned count;// = 0;
2579  const T* cur1 = vect1;
2580  const T* cur2 = vect2;
2581 
2582  unsigned bitval1 = (*cur1++ & 1);
2583  unsigned bitval2 = (*cur2++ & 1);
2584  unsigned bitval = count = f(bitval1, bitval2);
2585  unsigned bitval_prev = bitval;
2586 
2587  //if (bitval) ++count;
2588 
2589  T res, res_prev;
2590  res = res_prev = 0;
2591 
2592  while (1)
2593  {
2594  bitval = f(bitval1, bitval2);
2595 
2596  // Check if GAP value changes and we need to
2597  // start the next one.
2598  if (bitval != bitval_prev)
2599  {
2600  bitval_prev = bitval;
2601  res_prev = res;
2602  }
2603 
2604  if (*cur1 < *cur2)
2605  {
2606  res = *cur1;
2607  if (bitval)
2608  {
2609  count += res - res_prev;
2610  res_prev = res;
2611  }
2612  ++cur1; bitval1 ^= 1;
2613  }
2614  else // >=
2615  {
2616  res = *cur2;
2617  if (bitval)
2618  {
2619  count += res - res_prev;
2620  res_prev = res;
2621  }
2622  if (*cur2 < *cur1)
2623  {
2624  bitval2 ^= 1;
2625  }
2626  else // equal
2627  {
2628  if (*cur2 == (bm::gap_max_bits - 1))
2629  break;
2630 
2631  ++cur1;
2632  bitval1 ^= 1; bitval2 ^= 1;
2633  }
2634  ++cur2;
2635  }
2636 
2637  } // while
2638 
2639  return count;
2640 }
2641 
2642 
2643 #ifdef __GNUG__
2644 #pragma GCC diagnostic push
2645 #pragma GCC diagnostic ignored "-Wconversion"
2646 #endif
2647 
2648 /*!
2649  \brief Sets or clears bit in the GAP buffer.
2650 
2651  \param val - new bit value
2652  \param buf - GAP buffer.
2653  \param pos - Index of bit to set.
2654  \param is_set - (OUT) flag if bit was actually set.
2655 
2656  \return New GAP buffer length.
2657 
2658  @ingroup gapfunc
2659 */
2660 template<typename T>
2661 unsigned gap_set_value(unsigned val,
2662  T* BMRESTRICT buf,
2663  unsigned pos,
2664  unsigned* BMRESTRICT is_set) BMNOEXCEPT
2665 {
2666  BM_ASSERT(pos < bm::gap_max_bits);
2667 
2668  unsigned curr = bm::gap_bfind(buf, pos, is_set);
2669  T end = (T)(*buf >> 3);
2670  if (*is_set == val)
2671  {
2672  *is_set = 0;
2673  return end;
2674  }
2675  *is_set = 1;
2676 
2677  T* pcurr = buf + curr;
2678  T* pprev = pcurr - 1;
2679  T* pend = buf + end;
2680 
2681  // Special case, first bit GAP operation. There is no platform beside it.
2682  // initial flag must be inverted.
2683  if (!pos)
2684  {
2685  *buf ^= 1;
2686  if (buf[1]) // We need to insert a 1 bit GAP here
2687  {
2688  ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2689  buf[1] = 0;
2690  ++end;
2691  }
2692  else // Only 1 bit in the GAP. We need to delete the first GAP.
2693  {
2694  pprev = buf + 1; pcurr = pprev + 1;
2695  goto copy_gaps;
2696  }
2697  }
2698  else
2699  if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2700  {
2701  ++(*pprev);
2702  if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2703  {
2704  --end;
2705  if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2706  {
2707  ++pcurr;
2708  copy_gaps:
2709  --end;
2710  do { *pprev++ = *pcurr++; } while (pcurr < pend);
2711  }
2712  }
2713  }
2714  else
2715  if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2716  {
2717  --(*pcurr);
2718  end += (pcurr == pend);
2719  }
2720  else // Worst case: split current GAP
2721  {
2722  if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2723  ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2724  end += 2;
2725  pcurr[0] = (T)(pos-1);
2726  pcurr[1] = (T)pos;
2727  }
2728 
2729  // Set correct length word and last border word
2730  *buf = (T)((*buf & 7) + (end << 3));
2731  buf[end] = bm::gap_max_bits-1;
2732  return end;
2733 }
2734 
2735 /*!
2736  \brief Sets or clears bit in the GAP buffer.
2737 
2738  \param val - new bit value
2739  \param buf - GAP buffer.
2740  \param pos - Index of bit to set.
2741 
2742  \return New GAP buffer length.
2743 
2744  @ingroup gapfunc
2745 */
2746 template<typename T>
2747 unsigned gap_set_value(unsigned val,
2748  T* BMRESTRICT buf,
2749  unsigned pos) BMNOEXCEPT
2750 {
2751  BM_ASSERT(pos < bm::gap_max_bits);
2752  unsigned is_set;
2753  unsigned curr = bm::gap_bfind(buf, pos, &is_set);
2754  T end = (T)(*buf >> 3);
2755  if (is_set == val)
2756  return end;
2757 
2758  T* pcurr = buf + curr;
2759  T* pprev = pcurr - 1;
2760  T* pend = buf + end;
2761 
2762  // Special case, first bit GAP operation. There is no platform beside it.
2763  // initial flag must be inverted.
2764  if (!pos)
2765  {
2766  *buf ^= 1;
2767  if (buf[1]) // We need to insert a 1 bit GAP here
2768  {
2769  ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2770  buf[1] = 0;
2771  ++end;
2772  }
2773  else // Only 1 bit in the GAP. We need to delete the first GAP.
2774  {
2775  pprev = buf + 1; pcurr = pprev + 1;
2776  goto copy_gaps;
2777  }
2778  }
2779  else
2780  if (curr > 1 && ((unsigned)(*pprev))+1 == pos) // Left border bit
2781  {
2782  ++(*pprev);
2783  if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2784  {
2785  --end;
2786  if (pcurr != pend) // GAP merge: 2 GAPS to be deleted
2787  {
2788  ++pcurr;
2789  copy_gaps:
2790  --end;
2791  do { *pprev++ = *pcurr++; } while (pcurr < pend);
2792  }
2793  }
2794  }
2795  else
2796  if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2797  {
2798  --(*pcurr);
2799  end += (pcurr == pend);
2800  }
2801  else // Worst case: split current GAP
2802  {
2803  if (*pcurr != bm::gap_max_bits-1) // last gap does not need memmove
2804  ::memmove(pcurr+2, pcurr, (end - curr + 1)*(sizeof(T)));
2805  end += 2;
2806  pcurr[0] = (T)(pos-1);
2807  pcurr[1] = (T)pos;
2808  }
2809 
2810  // Set correct length word and last border word
2811  *buf = (T)((*buf & 7) + (end << 3));
2812  buf[end] = bm::gap_max_bits-1;
2813  return end;
2814 }
2815 
2816 /*!
2817  \brief Add new value to the end of GAP buffer.
2818 
2819  \param buf - GAP buffer.
2820  \param pos - Index of bit to set.
2821 
2822  \return New GAP buffer length.
2823 
2824  @ingroup gapfunc
2825 */
2826 template<typename T>
2827 unsigned gap_add_value(T* buf, unsigned pos) BMNOEXCEPT
2828 {
2829  BM_ASSERT(pos < bm::gap_max_bits);
2830 
2831  T end = (T)(*buf >> 3);
2832  T curr = end;
2833  T* pcurr = buf + end;
2834  T* pend = pcurr;
2835  T* pprev = pcurr - 1;
2836 
2837  // Special case, first bit GAP operation. There is no platform beside it.
2838  // initial flag must be inverted.
2839  if (!pos)
2840  {
2841  *buf ^= 1;
2842  if ( buf[1] ) // We need to insert a 1 bit platform here.
2843  {
2844  ::memmove(&buf[2], &buf[1], (end - 1) * sizeof(gap_word_t));
2845  buf[1] = 0;
2846  ++end;
2847  }
2848  else // Only 1 bit in the GAP. We need to delete the first GAP.
2849  {
2850  pprev = buf + 1; pcurr = pprev + 1;
2851  --end;
2852  do { *pprev++ = *pcurr++; } while (pcurr < pend);
2853  }
2854  }
2855  else if (((unsigned)(*pprev))+1 == pos && (curr > 1) ) // Left border bit
2856  {
2857  ++(*pprev);
2858  if (*pprev == *pcurr) // Curr. GAP to be merged with prev.GAP.
2859  {
2860  --end;
2861  BM_ASSERT(pcurr == pend);
2862  }
2863  }
2864  else if (*pcurr == pos) // Rightmost bit in the GAP. Border goes left.
2865  {
2866  --(*pcurr);
2867  end += (pcurr == pend);
2868  }
2869  else // Worst case we need to split current block.
2870  {
2871  pcurr[0] = (T)(pos-1);
2872  pcurr[1] = (T)pos;
2873  end = (T)(end+2);
2874  }
2875 
2876  // Set correct length word.
2877  *buf = (T)((*buf & 7) + (end << 3));
2878  buf[end] = bm::gap_max_bits - 1;
2879  return end;
2880 }
2881 
2882 #ifdef __GNUG__
2883 #pragma GCC diagnostic pop
2884 #endif
2885 
2886 
2887 /*!
2888  @brief Right shift GAP block by 1 bit
2889  @param buf - block pointer
2890  @param co_flag - carry over from the previous block
2891  @param new_len - output length of the GAP block after the operation
2892 
2893  @return carry over bit (1 or 0)
2894  @ingroup gapfunc
2895 */
2896 template<typename T>
2898  unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2899 {
2900  BM_ASSERT(new_len);
2901  bool co;
2902  // 1: increment all GAP values by 1
2903  {
2904  unsigned bitval = *buf & 1;
2905  if (buf[1] == bm::gap_max_bits-1) // full GAP block
2906  {
2907  co = bitval;
2908  }
2909  else
2910  {
2911  unsigned len = (*buf >> 3);
2912  unsigned i = 1;
2913  for (; i < len; ++i)
2914  {
2915  buf[i]++;
2916  bitval ^= 1;
2917  } // for i
2918  BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2919  if (buf[i-1] == bm::gap_max_bits-1) // last element shifts out
2920  {
2921  // Set correct length word
2922  --len;
2923  *buf = (T)((*buf & 7) + (len << 3));
2924  }
2925  co = bitval;
2926  }
2927  }
2928  // set bit position 0 with carry-in flag
2929  {
2930  unsigned is_set;
2931  *new_len = bm::gap_set_value(co_flag, buf, 0, &is_set);
2932  }
2933  return co;
2934 }
2935 
2936 /*!
2937  @brief Left shift GAP block by 1 bit
2938  @param buf - block pointer
2939  @param co_flag - carry over from the previous block
2940  @param new_len - new length of the block
2941 
2942  @return carry over bit (1 or 0)
2943  @ingroup gapfunc
2944 */
2945 template<typename T>
2947  unsigned co_flag, unsigned* BMRESTRICT new_len) BMNOEXCEPT
2948 {
2949  BM_ASSERT(new_len);
2950  unsigned is_set;
2951 
2952  // 1: decrement all GAP values by 1
2953  //
2954  unsigned bitval = *buf & 1;
2955  bool co0 = bitval;
2956 
2957  if (!buf[1]) // cannot decrement (corner case)
2958  {
2959  bitval ^= 1;
2960  *new_len = bm::gap_set_value(bitval, buf, 0, &is_set);
2961 
2962  BM_ASSERT(is_set);
2963  BM_ASSERT(buf[1]);
2964  BM_ASSERT(bitval == unsigned(*buf & 1u));
2965 
2966  if (*new_len == 1)
2967  {
2968  *new_len = bm::gap_set_value(co_flag, buf,
2969  bm::gap_max_bits-1, &is_set);
2970  return co0;
2971  }
2972  }
2973  if (buf[1] != bm::gap_max_bits-1) // full GAP block
2974  {
2975  BM_ASSERT(buf[1]);
2976  unsigned len = (*buf >> 3);
2977  unsigned i = 1;
2978  for (; i < len; ++i)
2979  {
2980  buf[i]--;
2981  bitval ^= 1;
2982  } // for i
2983  BM_ASSERT(buf[i] == bm::gap_max_bits-1);
2984  }
2985  // 2: set last bit position with carry-in flag
2986  //
2987  *new_len = bm::gap_set_value(co_flag, buf, bm::gap_max_bits-1, &is_set);
2988  return co0;
2989 }
2990 
2991 
2992 /*!
2993  \brief Convert array to GAP buffer.
2994 
2995  \param buf - GAP buffer.
2996  \param arr - array of values to set
2997  \param len - length of the array
2998 
2999  \return New GAP buffer length.
3000 
3001  @ingroup gapfunc
3002 */
3003 
3004 template<typename T>
3005 unsigned gap_set_array(T* buf, const T* arr, unsigned len) BMNOEXCEPT
3006 {
3007  *buf = (T)((*buf & 6u) + (1u << 3)); // gap header setup
3008 
3009  T* pcurr = buf + 1;
3010 
3011  unsigned i = 0;
3012  T curr = arr[i];
3013  if (curr != 0) // need to add the first gap: (0 to arr[0]-1)
3014  {
3015  *pcurr = (T)(curr - 1);
3016  ++pcurr;
3017  }
3018  else
3019  {
3020  ++(*buf); // GAP starts with 1
3021  }
3022  T prev = curr;
3023  T acc = prev;
3024 
3025  for (i = 1; i < len; ++i)
3026  {
3027  curr = arr[i];
3028  if (curr == prev + 1)
3029  {
3030  ++acc;
3031  prev = curr;
3032  }
3033  else
3034  {
3035  *pcurr++ = acc;
3036  acc = curr;
3037  *pcurr++ = (T)(curr-1);
3038  }
3039  prev = curr;
3040  }
3041  *pcurr = acc;
3042  if (acc != bm::gap_max_bits - 1)
3043  {
3044  ++pcurr;
3045  *pcurr = bm::gap_max_bits - 1;
3046  }
3047 
3048  unsigned gap_len = unsigned(pcurr - buf);
3049  BM_ASSERT(gap_len == ((gap_len << 3) >> 3));
3050 
3051  *buf = (T)((*buf & 7) + (gap_len << 3));
3052  return gap_len+1;
3053 }
3054 
3055 
3056 //------------------------------------------------------------------------
3057 
3058 /**
3059  \brief Compute number of GAPs in bit-array
3060  \param arr - array of BITs
3061  \param len - array length
3062 
3063  @ingroup gapfunc
3064 */
3065 template<typename T>
3066 unsigned bit_array_compute_gaps(const T* arr, unsigned len) BMNOEXCEPT
3067 {
3068  unsigned gap_count = 1;
3069  T prev = arr[0];
3070  if (prev > 0)
3071  ++gap_count;
3072  for (unsigned i = 1; i < len; ++i)
3073  {
3074  T curr = arr[i];
3075  if (curr != prev + 1)
3076  {
3077  gap_count += 2;
3078  }
3079  prev = curr;
3080  }
3081  return gap_count;
3082 }
3083 
3084 
3085 //------------------------------------------------------------------------
3086 
3087 /**
3088  \brief Searches for the next 1 bit in the GAP block
3089  \param buf - GAP buffer
3090  \param nbit - bit index to start checking from.
3091  \param prev - returns previously checked value
3092 
3093  \return 0 if not found
3094 
3095  @ingroup gapfunc
3096 */
3097 template<typename T>
3098 unsigned gap_block_find(const T* BMRESTRICT buf,
3099  unsigned nbit,
3101 {
3102  BM_ASSERT(nbit < bm::gap_max_bits);
3103 
3104  unsigned bitval;
3105  unsigned gap_idx = bm::gap_bfind(buf, nbit, &bitval);
3106 
3107  if (bitval) // positive block.
3108  {
3109  *prev = nbit;
3110  return 1u;
3111  }
3112  unsigned val = buf[gap_idx] + 1;
3113  *prev = val;
3114  return (val != bm::gap_max_bits); // no bug here.
3115 }
3116 
3117 //------------------------------------------------------------------------
3118 
3119 
3120 /*!
3121  \brief Set 1 bit in a block
3122  @ingroup bitfunc
3123 */
3125 void set_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3126 {
3127  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3128  unsigned nword = unsigned(nbit >> bm::set_word_shift);
3129  nbit &= bm::set_word_mask;
3130  dest[nword] |= unsigned(1u << nbit);
3131 }
3132 
3133 /*!
3134  \brief Set 1 bit in a block
3135  @ingroup bitfunc
3136 */
3138 void clear_bit(unsigned* dest, unsigned bitpos) BMNOEXCEPT
3139 {
3140  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3141  unsigned nword = unsigned(nbit >> bm::set_word_shift);
3142  nbit &= bm::set_word_mask;
3143  dest[nword] &= ~(unsigned(1u << nbit));
3144 }
3145 
3146 /*!
3147  \brief Test 1 bit in a block
3148 
3149  @ingroup bitfunc
3150 */
3152 unsigned test_bit(const unsigned* block, unsigned bitpos) BMNOEXCEPT
3153 {
3154  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3155  unsigned nword = unsigned(nbit >> bm::set_word_shift);
3156  nbit &= bm::set_word_mask;
3157  return (block[nword] >> nbit) & 1u;
3158 }
3159 
3160 
3161 /*!
3162  \brief Sets bits to 1 in the bitblock.
3163  \param dest - Bitset buffer.
3164  \param bitpos - Offset of the start bit.
3165  \param bitcount - number of bits to set.
3166 
3167  @ingroup bitfunc
3168 */
3169 inline
3170 void or_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3171 {
3172  const unsigned maskFF = ~0u;
3173 
3174  dest += unsigned(bitpos >> bm::set_word_shift); // nword
3175  bitpos &= bm::set_word_mask;
3176 
3177  if (bitcount == 1u) // special case (only 1 bit to set)
3178  {
3179  *dest |= (1u << bitpos);
3180  return;
3181  }
3182 
3183  if (bitpos) // starting pos is not aligned
3184  {
3185  unsigned mask_r = maskFF << bitpos;
3186  unsigned right_margin = bitpos + bitcount;
3187  if (right_margin < 32)
3188  {
3189  *dest |= (maskFF >> (32 - right_margin)) & mask_r;
3190  return;
3191  }
3192  *dest++ |= mask_r;
3193  bitcount -= 32 - bitpos;
3194  }
3195  for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3196  dest[0] = dest[1] = maskFF;
3197  if (bitcount >= 32)
3198  {
3199  *dest++ = maskFF; bitcount -= 32;
3200  }
3201  if (bitcount)
3202  {
3203  *dest |= maskFF >> (32 - bitcount);
3204  }
3205 }
3206 
3207 
3208 /*!
3209  \brief SUB (AND NOT) bit interval to 1 in the bitblock.
3210  \param dest - Bitset buffer.
3211  \param bitpos - Offset of the start bit.
3212  \param bitcount - number of bits to set.
3213 
3214  @ingroup bitfunc
3215 */
3216 inline
3217 void sub_bit_block(unsigned* dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
3218 {
3219  const unsigned maskFF = ~0u;
3220 
3221  dest += unsigned(bitpos >> bm::set_word_shift); // nword
3222  bitpos &= bm::set_word_mask;
3223 
3224  if (bitcount == 1u) // special case (only 1 bit to set)
3225  {
3226  *dest &= ~(1u << bitpos);
3227  return;
3228  }
3229 
3230  if (bitpos) // starting pos is not aligned
3231  {
3232  unsigned mask_r = maskFF << bitpos;
3233  unsigned right_margin = bitpos + bitcount;
3234  if (right_margin < 32)
3235  {
3236  *dest &= ~((maskFF >> (32 - right_margin)) & mask_r);
3237  return;
3238  }
3239  *dest++ &= ~mask_r;
3240  bitcount -= 32 - bitpos;
3241  }
3242  for ( ;bitcount >= 64; bitcount-=64, dest+=2)
3243  dest[0] = dest[1] = 0u;
3244  if (bitcount >= 32)
3245  {
3246  *dest++ = 0u; bitcount -= 32;
3247  }
3248  if (bitcount)
3249  {
3250  *dest &= ~(maskFF >> (32 - bitcount));
3251  }
3252 }
3253 
3254 
3255 
3256 /*!
3257  \brief XOR bit interval to 1 in the bitblock.
3258  \param dest - Bitset buffer.
3259  \param bitpos - Offset of the start bit.
3260  \param bitcount - number of bits to set.
3261 
3262  @ingroup bitfunc
3263 */
3264 inline void xor_bit_block(unsigned* dest,
3265  unsigned bitpos,
3266  unsigned bitcount) BMNOEXCEPT
3267 {
3268  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
3269  unsigned nword = unsigned(nbit >> bm::set_word_shift);
3270  nbit &= bm::set_word_mask;
3271 
3272  bm::word_t* word = dest + nword;
3273 
3274  if (bitcount == 1) // special case (only 1 bit to set)
3275  {
3276  *word ^= unsigned(1 << nbit);
3277  return;
3278  }
3279 
3280  if (nbit) // starting position is not aligned
3281  {
3282  unsigned right_margin = nbit + bitcount;
3283 
3284  // here we checking if we setting bits only in the current
3285  // word. Example: 00111000000000000000000000000000 (32 bits word)
3286 
3287  if (right_margin < 32)
3288  {
3289  unsigned mask =
3291  bm::block_set_table<true>::_left[right_margin-1];
3292  *word ^= mask;
3293  return;
3294  }
3295  *word ^= block_set_table<true>::_right[nbit];
3296  bitcount -= 32 - nbit;
3297  ++word;
3298  }
3299  for ( ;bitcount >= 64; bitcount-=64, word+=2)
3300  {
3301  word[0] ^= ~0u; word[1] ^= ~0u;
3302  }
3303  if (bitcount >= 32)
3304  {
3305  *word++ ^= ~0u; bitcount -= 32;
3306  }
3307  if (bitcount)
3308  *word ^= block_set_table<true>::_left[bitcount-1];
3309 }
3310 
3311 
3312 /*!
3313  \brief SUB (AND NOT) GAP block to bitblock.
3314  \param dest - bitblock buffer pointer.
3315  \param pcurr - GAP buffer pointer.
3316 
3317  @ingroup gapfunc
3318 */
3319 template<typename T>
3320 void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3321  const T* BMRESTRICT pcurr) BMNOEXCEPT
3322 {
3323  BM_ASSERT(dest && pcurr);
3324 
3325  const T* pend = pcurr + (*pcurr >> 3);
3326  if (*pcurr & 1) // Starts with 1
3327  {
3328  bm::sub_bit_block(dest, 0, 1 + pcurr[1]);
3329  ++pcurr;
3330  }
3331  for (pcurr += 2; pcurr <= pend; pcurr += 2)
3332  {
3333  BM_ASSERT(*pcurr > pcurr[-1]);
3334  bm::sub_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3335  }
3336 }
3337 
3338 
3339 /*!
3340  \brief SUB (AND NOT) GAP block to bitblock with digest assist
3341 
3342  \param dest - bitblock buffer pointer.
3343  \param pcurr - GAP buffer pointer.
3344  \param digest0 - digest of 0 strides inside bit block
3345 
3346  @ingroup gapfunc
3347 */
3348 template<typename T>
3349 void gap_sub_to_bitset(unsigned* BMRESTRICT dest,
3350  const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3351 {
3352  BM_ASSERT(dest && pcurr);
3353 
3354  const T* pend = pcurr + (*pcurr >> 3);
3355  if (*pcurr & 1) // Starts with 1
3356  {
3357  bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3358  if (!all_zero)
3359  bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3360  pcurr += 3;
3361  }
3362  else
3363  pcurr += 2;
3364 
3365  // wind forward to digest start
3366  {
3367  unsigned tz = bm::count_trailing_zeros_u64(digest0);
3368  unsigned start_pos = tz << set_block_digest_pos_shift;
3369  for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3370  {
3371  if (*pcurr >= start_pos)
3372  break;
3373  }
3374  }
3375 
3376  unsigned lz = bm::count_leading_zeros_u64(digest0);
3377  unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3378 
3379  unsigned bc, pos;
3380  T prev;
3381  for (; pcurr <= pend; pcurr += 2) // now we are in GAP "1" again
3382  {
3383  BM_ASSERT(*pcurr > *(pcurr-1));
3384  prev = pcurr[-1];
3385  bc = *pcurr - prev;
3386  pos = 1u + prev;
3387 
3388  bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3389  if (!all_zero)
3390  bm::sub_bit_block(dest, pos, bc);
3391 
3392  if (pos > stop_pos)
3393  break; // early break is possible based on digest tail
3394 
3395  } // for
3396 }
3397 
3398 
3399 
3400 /*!
3401  \brief XOR GAP block to bitblock.
3402  \param dest - bitblock buffer pointer.
3403  \param pcurr - GAP buffer pointer.
3404 
3405  @ingroup gapfunc
3406 */
3407 template<typename T>
3408 void gap_xor_to_bitset(unsigned* BMRESTRICT dest,
3409  const T* BMRESTRICT pcurr) BMNOEXCEPT
3410 {
3411  BM_ASSERT(dest && pcurr);
3412 
3413  const T* pend = pcurr + (*pcurr >> 3);
3414  if (*pcurr & 1) // Starts with 1
3415  {
3416  bm::xor_bit_block(dest, 0, 1 + pcurr[1]);
3417  ++pcurr;
3418  }
3419  for (pcurr += 2; pcurr <= pend; pcurr += 2)
3420  {
3421  BM_ASSERT(*pcurr > pcurr[-1]);
3422  bm::xor_bit_block(dest, 1 + pcurr[-1], *pcurr - pcurr[-1]);
3423  }
3424 }
3425 
3426 
3427 /*!
3428  \brief Adds(OR) GAP block to bitblock.
3429  \param dest - bitblock buffer pointer.
3430  \param pcurr - GAP buffer pointer.
3431  \param len - gap length
3432 
3433  @ingroup gapfunc
3434 */
3435 template<typename T>
3436 void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3437  const T* BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
3438 {
3439  BM_ASSERT(dest && pcurr);
3440 
3441  const T* pend = pcurr + len;
3442  if (*pcurr & 1) // Starts with 1
3443  {
3444  bm::or_bit_block(dest, 0, 1 + pcurr[1]);
3445  pcurr += 3;
3446  }
3447  else
3448  pcurr += 2;
3449 
3450  unsigned bc, pos;
3451  for (; pcurr <= pend; )
3452  {
3453  BM_ASSERT(*pcurr > pcurr[-1]);
3454  pos = 1u + pcurr[-1];
3455  bc = *pcurr - pcurr[-1];
3456  pcurr += 2;
3457  bm::or_bit_block(dest, pos, bc);
3458  }
3459 }
3460 
3461 
3462 /*!
3463  \brief Adds(OR) GAP block to bitblock.
3464  \param dest - bitblock buffer pointer.
3465  \param pcurr - GAP buffer pointer.
3466 
3467  @ingroup gapfunc
3468 */
3469 template<typename T>
3470 void gap_add_to_bitset(unsigned* BMRESTRICT dest,
3471  const T* BMRESTRICT pcurr) BMNOEXCEPT
3472 {
3473  unsigned len = (*pcurr >> 3);
3474  gap_add_to_bitset(dest, pcurr, len);
3475 }
3476 
3477 
3478 /*!
3479  \brief ANDs GAP block to bitblock.
3480  \param dest - bitblock buffer pointer.
3481  \param pcurr - GAP buffer pointer.
3482 
3483  @ingroup gapfunc
3484 */
3485 template<typename T>
3486 void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3487  const T* BMRESTRICT pcurr) BMNOEXCEPT
3488 {
3489  BM_ASSERT(dest && pcurr);
3490 
3491  const T* pend = pcurr + (*pcurr >> 3);
3492  if (!(*pcurr & 1) ) // Starts with 0
3493  {
3494  bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3495  pcurr += 3;
3496  }
3497  else
3498  pcurr += 2;
3499 
3500  unsigned bc, pos;
3501  for (; pcurr <= pend; ) // now we are in GAP "0" again
3502  {
3503  BM_ASSERT(*pcurr > *(pcurr-1));
3504  pos = 1u + pcurr[-1];
3505  bc = *pcurr - pcurr[-1];
3506  pcurr += 2;
3507  bm::sub_bit_block(dest, pos, bc);
3508  }
3509 }
3510 
3511 
3512 /*!
3513  \brief ANDs GAP block to bitblock with digest assist
3514  \param dest - bitblock buffer pointer.
3515  \param pcurr - GAP buffer pointer.
3516  \param digest0 - digest of 0 strides for the destination
3517 
3518  @ingroup gapfunc
3519 */
3520 template<typename T>
3521 void gap_and_to_bitset(unsigned* BMRESTRICT dest,
3522  const T* BMRESTRICT pcurr, bm::id64_t digest0) BMNOEXCEPT
3523 {
3524  BM_ASSERT(dest && pcurr);
3525  if (!digest0)
3526  return;
3527 
3528  const T* pend = pcurr + (*pcurr >> 3);
3529  if (!(*pcurr & 1) ) // Starts with 0
3530  {
3531  bool all_zero = bm::check_zero_digest(digest0, 0, pcurr[1]+1);
3532  if (!all_zero)
3533  bm::sub_bit_block(dest, 0, pcurr[1] + 1); // (not AND) - SUB [0] gaps
3534  pcurr += 3;
3535  }
3536  else
3537  pcurr += 2;
3538 
3539  // wind forward to digest start
3540  {
3541  unsigned tz = bm::count_trailing_zeros_u64(digest0);
3542  unsigned start_pos = tz << set_block_digest_pos_shift;
3543  for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0"
3544  {
3545  if (*pcurr >= start_pos)
3546  break;
3547  }
3548  }
3549 
3550  unsigned lz = bm::count_leading_zeros_u64(digest0);
3551  unsigned stop_pos = (64u - lz) << set_block_digest_pos_shift;
3552 
3553  unsigned bc, pos;
3554  T prev;
3555  for (; pcurr <= pend; pcurr += 2) // now we are in GAP "0" again
3556  {
3557  BM_ASSERT(*pcurr > *(pcurr-1));
3558 
3559  prev = pcurr[-1];
3560  bc = *pcurr - prev;
3561  pos = 1u + prev;
3562 
3563  bool all_zero = bm::check_zero_digest(digest0, prev, *pcurr);
3564  if (!all_zero)
3565  bm::sub_bit_block(dest, pos, bc);
3566 
3567  if (pos > stop_pos) // early break is possible based on digest tail
3568  break;
3569 
3570  } // for
3571 }
3572 
3573 
3574 /*!
3575  \brief Compute bitcount of bit block AND masked by GAP block
3576  \param block - bitblock buffer pointer
3577  \param pcurr - GAP buffer pointer
3578  \return bitcount - cardinality of the AND product
3579 
3580  @ingroup gapfunc
3581 */
3582 template<typename T>
3584  const T* BMRESTRICT pcurr) BMNOEXCEPT
3585 {
3586  BM_ASSERT(block);
3587  const T* pend = pcurr + (*pcurr >> 3);
3588  bm::id_t count = 0;
3589  if (*pcurr & 1) // Starts with 1
3590  {
3591  count += bm::bit_block_calc_count_range(block, 0, pcurr[1]);
3592  ++pcurr;
3593  }
3594  for (pcurr +=2 ;pcurr <= pend; pcurr += 2)
3595  {
3596  count += bm::bit_block_calc_count_range(block, pcurr[-1]+1, *pcurr);
3597  }
3598  return count;
3599 }
3600 
3601 
3602 /*!
3603  \brief Bitcount test of bit block AND masked by GAP block.
3604  \param block - bitblock buffer pointer
3605  \param pcurr - GAP buffer pointer
3606  \return non-zero value if AND produces any result
3607 
3608  @ingroup gapfunc
3609 */
3610 template<typename T>
3612  const T* BMRESTRICT pcurr) BMNOEXCEPT
3613 {
3614  BM_ASSERT(block);
3615 
3616  const T* pend = pcurr + (*pcurr >> 3);
3617  bm::id_t count = 0;
3618  if (*pcurr & 1) // Starts with 1
3619  {
3620  count = bm::bit_block_any_range(block, 0, pcurr[1]);
3621  ++pcurr;
3622  }
3623  for (pcurr +=2 ;!count && pcurr <= pend; pcurr += 2)
3624  {
3625  count = bm::bit_block_any_range(block, pcurr[-1]+1, *pcurr);
3626  }
3627  return count;
3628 }
3629 
3630 
3631 
3632 /*!
3633  \brief Compute bitcount of bit block SUB masked by GAP block.
3634  \param block - bitblock buffer pointer.
3635  \param buf - GAP buffer pointer.
3636  \return bit-count result of AND NOT operation
3637 
3638  @ingroup gapfunc
3639 */
3640 template<typename T>
3642  const T* BMRESTRICT buf) BMNOEXCEPT
3643 {
3644  BM_ASSERT(block);
3645 
3646  const T* pcurr = buf;
3647  const T* pend = pcurr + (*pcurr >> 3);
3648  ++pcurr;
3649 
3650  bm::id_t count = 0;
3651 
3652  if (!(*buf & 1)) // Starts with 0
3653  {
3654  count += bit_block_calc_count_range(block, 0, *pcurr);
3655  ++pcurr;
3656  }
3657  ++pcurr; // now we are in GAP "0" again
3658 
3659  for (;pcurr <= pend; pcurr+=2)
3660  {
3661  count += bm::bit_block_calc_count_range(block, *(pcurr-1)+1, *pcurr);
3662  }
3663  return count;
3664 }
3665 
3666 
3667 /*!
3668  \brief Compute bitcount test of bit block SUB masked by GAP block
3669  \param block - bitblock buffer pointer
3670  \param buf - GAP buffer pointer
3671  \return non-zero value if AND NOT produces any 1 bits
3672 
3673  @ingroup gapfunc
3674 */
3675 template<typename T>
3677  const T* BMRESTRICT buf) BMNOEXCEPT
3678 {
3679  BM_ASSERT(block);
3680 
3681  const T* pcurr = buf;
3682  const T* pend = pcurr + (*pcurr >> 3);
3683  ++pcurr;
3684 
3685  bm::id_t count = 0;
3686 
3687  if (!(*buf & 1)) // Starts with 0
3688  {
3689  count += bit_block_any_range(block, 0, *pcurr);
3690  if (count)
3691  return count;
3692  ++pcurr;
3693  }
3694  ++pcurr; // now we are in GAP "0" again
3695 
3696  for (; !count && pcurr <= pend; pcurr+=2)
3697  {
3698  count += bm::bit_block_any_range(block, *(pcurr-1)+1, *pcurr);
3699  }
3700  return count;
3701 }
3702 
3703 
3704 
3705 /*!
3706  \brief Compute bitcount of bit block XOR masked by GAP block
3707  \param block - bitblock buffer pointer
3708  \param buf - GAP buffer pointer
3709  \return bit count value of XOR operation
3710 
3711  @ingroup gapfunc
3712 */
3713 template<typename T>
3715  const T* BMRESTRICT buf) BMNOEXCEPT
3716 {
3717  BM_ASSERT(block);
3718 
3719  const T* pcurr = buf;
3720  const T* pend = pcurr + (*pcurr >> 3);
3721  ++pcurr;
3722 
3723  unsigned bitval = *buf & 1;
3724 
3725  bm::id_t count = bm::bit_block_calc_count_range(block, 0, *pcurr);
3726  if (bitval)
3727  {
3728  count = *pcurr + 1 - count;
3729  }
3730 
3731  for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3732  {
3733  T prev = (T)(*(pcurr-1)+1);
3734  bm::id_t c = bit_block_calc_count_range(block, prev, *pcurr);
3735 
3736  if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3737  c = (*pcurr - prev + 1) - c;
3738  count += c;
3739  }
3740  return count;
3741 }
3742 
3743 /*!
3744  \brief Compute bitcount test of bit block XOR masked by GAP block.
3745  \param block - bitblock buffer pointer
3746  \param buf - GAP buffer pointer
3747  \return non-zero value if XOR returns anything
3748 
3749  @ingroup gapfunc
3750 */
3751 template<typename T>
3753  const T* BMRESTRICT buf) BMNOEXCEPT
3754 {
3755  BM_ASSERT(block);
3756 
3757  const T* pcurr = buf;
3758  const T* pend = pcurr + (*pcurr >> 3);
3759  ++pcurr;
3760 
3761  unsigned bitval = *buf & 1;
3762 
3763  bm::id_t count = bit_block_any_range(block, 0, *pcurr);
3764  if (bitval)
3765  count = *pcurr + 1 - count;
3766 
3767  for (bitval^=1, ++pcurr; !count && pcurr <= pend; bitval^=1, ++pcurr)
3768  {
3769  T prev = (T)(*(pcurr-1)+1);
3770  bm::id_t c = bit_block_any_range(block, prev, *pcurr);
3771 
3772  if (bitval) // 1 gap; means Result = Total_Bits - BitCount;
3773  c = (*pcurr - prev + 1) - c;
3774  count += c;
3775  }
3776  return count;
3777 }
3778 
3779 
3780 
3781 /*!
3782  \brief Compute bitcount of bit block OR masked by GAP block.
3783  \param block - bitblock buffer pointer.
3784  \param buf - GAP buffer pointer.
3785  \return bit count of OR
3786 
3787  @ingroup gapfunc
3788 */
3789 template<typename T>
3791  const T* BMRESTRICT buf) BMNOEXCEPT
3792 {
3793  BM_ASSERT(block);
3794  const T* pcurr = buf;
3795  const T* pend = pcurr + (*pcurr >> 3);
3796  ++pcurr;
3797 
3798  unsigned bitval = *buf & 1;
3799 
3800  bm::id_t count = bitval ? *pcurr + 1
3801  : bm::bit_block_calc_count_range(block, 0, *pcurr);
3802  for (bitval^=1, ++pcurr; pcurr <= pend; bitval^=1, ++pcurr)
3803  {
3804  T prev = (T)(*(pcurr-1)+1);
3805  bm::id_t c =
3806  bitval ? (*pcurr - prev + 1)
3807  : bm::bit_block_calc_count_range(block, prev, *pcurr);
3808  count += c;
3809  }
3810  return count;
3811 }
3812 
3813 /*!
3814  \brief Compute bitcount test of bit block OR masked by GAP block
3815  \param block - bitblock buffer pointer
3816  \param buf - GAP buffer pointer
3817  \return non zero value if union (OR) returns anything
3818 
3819  @ingroup gapfunc
3820 */
3821 template<typename T>
3822 bm::id_t gap_bitset_or_any(const unsigned* BMRESTRICT block,
3823  const T* BMRESTRICT buf) BMNOEXCEPT
3824 {
3825  bool b = !bm::gap_is_all_zero(buf) ||
3826  !bm::bit_is_all_zero(block);
3827  return b;
3828 }
3829 
3830 
3831 
3832 /*!
3833  \brief Bitblock memset operation.
3834 
3835  \param dst - destination block.
3836  \param value - value to set.
3837 
3838  @ingroup bitfunc
3839 */
3840 inline
3842 {
3843 #ifdef BMVECTOPT
3844  VECT_SET_BLOCK(dst, value);
3845 #else
3846  ::memset(dst, int(value), bm::set_block_size * sizeof(bm::word_t));
3847 #endif
3848 }
3849 
3850 
3851 /*!
3852  \brief GAP block to bitblock conversion.
3853  \param dest - bitblock buffer pointer.
3854  \param buf - GAP buffer pointer.
3855 
3856  @ingroup gapfunc
3857 */
3858 template<typename T>
3859 void gap_convert_to_bitset(unsigned* BMRESTRICT dest,
3860  const T* BMRESTRICT buf) BMNOEXCEPT
3861 {
3862  bm::bit_block_set(dest, 0);
3863  bm::gap_add_to_bitset(dest, buf);
3864 }
3865 
3866 
3867 
3868 /*!
3869  \brief Smart GAP block to bitblock conversion.
3870 
3871  Checks if GAP block is ALL-ZERO or ALL-ON. In those cases returns
3872  pointer on special static bitblocks.
3873 
3874  \param dest - bitblock buffer pointer.
3875  \param buf - GAP buffer pointer.
3876  \param set_max - max possible bitset length
3877 
3878  @ingroup gapfunc
3879 */
3880 template<typename T>
3881 unsigned* gap_convert_to_bitset_smart(unsigned* BMRESTRICT dest,
3882  const T* BMRESTRICT buf,
3883  id_t set_max) BMNOEXCEPT
3884 {
3885  if (buf[1] == set_max - 1)
3886  return (buf[0] & 1) ? FULL_BLOCK_REAL_ADDR : 0;
3887  bm::gap_convert_to_bitset(dest, buf);
3888  return dest;
3889 }
3890 
3891 
3892 /*!
3893  \brief Calculates sum of all words in GAP block. (For debugging purposes)
3894  \note For debugging and testing ONLY.
3895  \param buf - GAP buffer pointer.
3896  \return Sum of all words.
3897 
3898  @ingroup gapfunc
3899  @internal
3900 */
3901 template<typename T>
3902 unsigned gap_control_sum(const T* buf) BMNOEXCEPT
3903 {
3904  unsigned end = *buf >> 3;
3905 
3906  const T* pcurr = buf;
3907  const T* pend = pcurr + (*pcurr >> 3);
3908  ++pcurr;
3909 
3910  if (*buf & 1) // Starts with 1
3911  {
3912  ++pcurr;
3913  }
3914  ++pcurr; // now we are in GAP "1" again
3915  while (pcurr <= pend)
3916  {
3917  BM_ASSERT(*pcurr > *(pcurr-1));
3918  pcurr += 2;
3919  }
3920  return buf[end];
3921 }
3922 
3923 
3924 /*!
3925  \brief Sets all bits to 0 or 1 (GAP)
3926  \param buf - GAP buffer pointer.
3927  \param set_max - max possible bitset length
3928  \param value - value to set
3929 
3930  @ingroup gapfunc
3931 */
3932 template<class T>
3933 void gap_set_all(T* buf, unsigned set_max, unsigned value) BMNOEXCEPT
3934 {
3935  BM_ASSERT(value == 0 || value == 1);
3936  *buf = (T)((*buf & 6u) + (1u << 3) + value);
3937  *(++buf) = (T)(set_max - 1);
3938 }
3939 
3940 
3941 /*!
3942  \brief Init gap block so it has block in it (can be whole block)
3943  \param buf - GAP buffer pointer.
3944  \param from - one block start
3945  \param to - one block end
3946  \param value - (block value)1 or 0
3947 
3948  @ingroup gapfunc
3949 */
3950 template<class T>
3952  T from,
3953  T to,
3954  T value) BMNOEXCEPT
3955 {
3956  BM_ASSERT(value == 0 || value == 1);
3957  const unsigned set_max = bm::bits_in_block;
3958 
3959  unsigned gap_len;
3960  if (from == 0)
3961  {
3962  if (to == set_max - 1)
3963  {
3964  bm::gap_set_all(buf, set_max, value);
3965  }
3966  else
3967  {
3968  gap_len = 2;
3969  buf[1] = (T)to;
3970  buf[2] = (T)(set_max - 1);
3971  buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3972  }
3973  return;
3974  }
3975  // from != 0
3976 
3977  value = !value;
3978  if (to == set_max - 1)
3979  {
3980  gap_len = 2;
3981  buf[1] = (T)(from - 1);
3982  buf[2] = (T)(set_max - 1);
3983  }
3984  else
3985  {
3986  gap_len = 3;
3987  buf[1] = (T) (from - 1);
3988  buf[2] = (T) to;
3989  buf[3] = (T)(set_max - 1);
3990  }
3991  buf[0] = (T)((*buf & 6u) + (gap_len << 3) + value);
3992 }
3993 
3994 
3995 /*!
3996  \brief Inverts all bits in the GAP buffer.
3997  \param buf - GAP buffer pointer.
3998 
3999  @ingroup gapfunc
4000 */
4001 template<typename T> void gap_invert(T* buf) BMNOEXCEPT
4002 {
4003  *buf ^= 1;
4004 }
4005 
4006 
4007 #ifdef __GNUG__
4008 #pragma GCC diagnostic push
4009 #pragma GCC diagnostic ignored "-Wconversion"
4010 #endif
4011 
4012 /*!
4013  \brief Sets GAP block capacity level.
4014  \param buf - GAP buffer pointer.
4015  \param level new GAP block capacity level.
4016 
4017  @ingroup gapfunc
4018 */
4019 template<typename T>
4020 void set_gap_level(T* buf, int level) BMNOEXCEPT
4021 {
4022  BM_ASSERT(level >= 0);
4023  BM_ASSERT(unsigned(level) < bm::gap_levels);
4024 
4025  *buf = (T)(((level & 3) << 1) | (*buf & 1) | (*buf & ~7));
4026 }
4027 #ifdef __GNUG__
4028 #pragma GCC diagnostic pop
4029 #endif
4030 
4031 
4032 
4033 /*!
4034  \brief Calculates GAP block capacity level.
4035  \param len - GAP buffer length.
4036  \param glevel_len - GAP lengths table
4037  \return GAP block capacity level.
4038  -1 if block does not fit any level.
4039  @ingroup gapfunc
4040 */
4041 template<typename T>
4042 int gap_calc_level(unsigned len, const T* glevel_len) BMNOEXCEPT
4043 {
4044  if (len <= unsigned(glevel_len[0]-4)) return 0;
4045  if (len <= unsigned(glevel_len[1]-4)) return 1;
4046  if (len <= unsigned(glevel_len[2]-4)) return 2;
4047  if (len <= unsigned(glevel_len[3]-4)) return 3;
4048 
4049  BM_ASSERT(bm::gap_levels == 4);
4050  return -1;
4051 }
4052 
4053 /*! @brief Returns number of free elements in GAP block array.
4054  Difference between GAP block capacity on this level and actual GAP length.
4055 
4056  @param buf - GAP buffer pointer
4057  @param glevel_len - GAP lengths table
4058 
4059  @return Number of free GAP elements
4060  @ingroup gapfunc
4061 */
4062 template<typename T>
4063 inline unsigned gap_free_elements(const T* BMRESTRICT buf,
4064  const T* BMRESTRICT glevel_len) BMNOEXCEPT
4065 {
4066  unsigned len = bm::gap_length(buf);
4067  unsigned capacity = bm::gap_capacity(buf, glevel_len);
4068  return capacity - len;
4069 }
4070 
4071 /*!
4072  \brief Lexicographical comparison of BIT buffers.
4073  \param buf1 - First buffer pointer.
4074  \param buf2 - Second buffer pointer.
4075  \param len - Buffer length in elements (T).
4076  \return <0 - less, =0 - equal, >0 - greater.
4077 
4078  @ingroup bitfunc
4079 */
4080 template<typename T>
4081 int bitcmp(const T* buf1, const T* buf2, unsigned len) BMNOEXCEPT
4082 {
4083  BM_ASSERT(len);
4084  const T* pend1 = buf1 + len;
4085  do
4086  {
4087  T w1 = *buf1++;
4088  T w2 = *buf2++;
4089  T diff = w1 ^ w2;
4090  if (diff)
4091  return (w1 & diff & -diff) ? 1 : -1;
4092  } while (buf1 < pend1);
4093  return 0;
4094 }
4095 
4096 /*!
4097  \brief Find first bit which is different between two bit-blocks
4098  \param blk1 - block 1
4099  \param blk2 - block 2
4100  \param pos - out - position of difference (undefined if blocks are equal)
4101  \return true if difference was found
4102 
4103  @ingroup bitfunc
4104 */
4105 inline
4107  const bm::word_t* BMRESTRICT blk2,
4108  unsigned* BMRESTRICT pos) BMNOEXCEPT
4109 {
4110  BM_ASSERT(blk1 && blk2 && pos);
4111 #ifdef VECT_BIT_FIND_DIFF
4112  bool f = VECT_BIT_FIND_DIFF(blk1, blk2, pos);
4113  return f;
4114 #else
4115 #ifdef BM64OPT
4116  BM_ASSERT(sizeof(bm::wordop_t) == 8);
4117 
4118  const bm::wordop_t* b1 = (const bm::wordop_t*) blk1;
4119  const bm::wordop_t* b2 = (const bm::wordop_t*) blk2;
4120 
4121  for (unsigned i = 0; i < bm::set_block_size/2; ++i)
4122  {
4123  bm::wordop_t w1 = b1[i]; bm::wordop_t w2 = b2[i];
4124  bm::wordop_t diff = w1 ^ w2;
4125  if (diff)
4126  {
4127  unsigned idx = bm::count_trailing_zeros_u64(diff);
4128  *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::wordop_t))));
4129  return true;
4130  }
4131  } // for
4132 #else
4133  for (unsigned i = 0; i < bm::set_block_size; ++i)
4134  {
4135  bm::word_t w1 = blk1[i]; bm::word_t w2 = blk2[i];
4136  bm::word_t diff = w1 ^ w2;
4137  if (diff)
4138  {
4139  unsigned idx = bm::bit_scan_forward32(diff); // trailing zeros
4140  *pos = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
4141  return true;
4142  }
4143  } // for
4144 #endif
4145 #endif
4146  return false;
4147 }
4148 
4149 
4150 #ifndef BMAVX2OPT
4151 
4152 /*!
4153  \brief Converts bit block to GAP.
4154  \param dest - Destinatio GAP buffer.
4155  \param block - Source bitblock buffer.
4156  \param dest_len - length of the destination buffer.
4157  \return New length of GAP block or 0 if conversion failed
4158  (insufficicent space).
4159 
4160  @ingroup gapfunc
4161 */
4162 inline
4164  const unsigned* BMRESTRICT block,
4165  unsigned dest_len) BMNOEXCEPT
4166 {
4167  const unsigned* BMRESTRICT block_end = block + bm::set_block_size;
4168  gap_word_t* BMRESTRICT pcurr = dest;
4169  gap_word_t* BMRESTRICT end = dest + dest_len; (void)end;
4170 
4171  unsigned bitval = (*block) & 1u;
4172  *pcurr++ = bm::gap_word_t(bitval);
4173  *pcurr = 0;
4174  unsigned bit_idx = 0;
4175 
4176  do
4177  {
4178  unsigned val = *block;
4179  while (!val || val == ~0u)
4180  {
4181  if (bitval != unsigned(bool(val)))
4182  {
4183  *pcurr++ = (gap_word_t)(bit_idx-1);
4184  bitval ^= 1u;
4185  BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4186  BM_ASSERT(pcurr != end);
4187  }
4188  bit_idx += unsigned(sizeof(*block) * 8);
4189  if (++block >= block_end)
4190  goto complete;
4191  val = *block;
4192  } // while
4193 
4194  // process "0100011" word
4195  //
4196  unsigned bits_consumed = 0;
4197  do
4198  {
4199  unsigned tz = 1u;
4200  if (bitval != (val & 1u))
4201  {
4202  *pcurr++ = (gap_word_t)(bit_idx-1);
4203  bitval ^= 1u;
4204  BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4205  BM_ASSERT(pcurr != end);
4206  }
4207  else // match, find the next idx
4208  {
4209  tz = bm::bit_scan_forward32(bitval ? ~val : val);
4210  // possible alternative:
4211  // tz = bm::count_trailing_zeros(bitval ? ~val : val);
4212  }
4213 
4214  bits_consumed += tz;
4215  bit_idx += tz;
4216  val >>= tz;
4217 
4218  if (!val)
4219  {
4220  if (bits_consumed < 32u)
4221  {
4222  *pcurr++ = (gap_word_t)(bit_idx-1);
4223  bitval ^= 1u;
4224  bit_idx += 32u - bits_consumed;
4225  BM_ASSERT((pcurr-1) == (dest+1) || *(pcurr-1) > *(pcurr-2));
4226  BM_ASSERT(pcurr != end);
4227  }
4228  break;
4229  }
4230  } while (1);
4231 
4232  } while(++block < block_end);
4233 
4234 complete:
4235  *pcurr = (gap_word_t)(bit_idx-1);
4236  unsigned len = (unsigned)(pcurr - dest);
4237  *dest = (gap_word_t)((*dest & 7) + (len << 3));
4238  return len;
4239 }
4240 #endif
4241 
4242 /**
4243  Convert bit block to GAP representation
4244  @internal
4245  @ingroup bitfunc
4246 */
4247 inline
4249  const unsigned* BMRESTRICT block,
4250  unsigned dest_len) BMNOEXCEPT
4251 {
4252 #if defined(VECT_BIT_TO_GAP)
4253  return VECT_BIT_TO_GAP(dest, block, dest_len);
4254 #else
4255  return bm::bit_block_to_gap(dest, block, dest_len);
4256 #endif
4257 }
4258 
4259 
4260 /*!
4261  \brief Iterate gap block as delta-bits with a functor
4262  @ingroup gapfunc
4263 */
4264 template<class T, class F>
4265 void for_each_gap_dbit(const T* buf, F& func)
4266 {
4267  const T* pcurr = buf;
4268  const T* pend = pcurr + (*pcurr >> 3);
4269 
4270  ++pcurr;
4271 
4272  unsigned prev = 0;
4273  unsigned first_inc;
4274 
4275  if (*buf & 1)
4276  {
4277  first_inc = 0;
4278  unsigned to = *pcurr;
4279  for (unsigned i = 0; i <= to; ++i)
4280  {
4281  func(1);
4282  }
4283  prev = to;
4284  ++pcurr;
4285  }
4286  else
4287  {
4288  first_inc = 1;
4289  }
4290  ++pcurr; // set GAP to 1
4291 
4292  while (pcurr <= pend)
4293  {
4294  unsigned from = *(pcurr-1)+1;
4295  unsigned to = *pcurr;
4296  if (first_inc)
4297  {
4298  func(from - prev + first_inc);
4299  first_inc = 0;
4300  }
4301  else
4302  {
4303  func(from - prev);
4304  }
4305 
4306  for (unsigned i = from+1; i <= to; ++i)
4307  {
4308  func(1);
4309  }
4310  prev = to;
4311  pcurr += 2; // jump to the next positive GAP
4312  }
4313 }
4314 
4315 /*!
4316  \brief Convert gap block into array of ints corresponding to 1 bits
4317  @ingroup gapfunc
4318 */
4319 template<typename D, typename T>
4321  const T* BMRESTRICT buf,
4322  unsigned dest_len,
4323  bool invert = false) BMNOEXCEPT
4324 {
4325  const T* BMRESTRICT pcurr = buf;
4326  const T* pend = pcurr + (*pcurr >> 3);
4327 
4328  D* BMRESTRICT dest_curr = dest;
4329  ++pcurr;
4330 
4331  int bitval = (*buf) & 1;
4332  if (invert)
4333  bitval = !bitval; // invert the GAP buffer
4334 
4335  if (bitval)
4336  {
4337  if (unsigned(*pcurr + 1) >= dest_len)
4338  return 0; // insufficient space
4339  dest_len -= *pcurr;
4340  T to = *pcurr;
4341  for (T i = 0; ;++i)
4342  {
4343  *dest_curr++ = i;
4344  if (i == to) break;
4345  }
4346  ++pcurr;
4347  }
4348  ++pcurr; // set GAP to 1
4349 
4350  while (pcurr <= pend)
4351  {
4352  unsigned pending = *pcurr - *(pcurr-1);
4353  if (pending >= dest_len)
4354  return 0;
4355  dest_len -= pending;
4356  T from = (T)(*(pcurr-1)+1);
4357  T to = *pcurr;
4358  for (T i = from; ;++i)
4359  {
4360  *dest_curr++ = i;
4361  if (i == to) break;
4362  }
4363  pcurr += 2; // jump to the next positive GAP
4364  }
4365  return (D) (dest_curr - dest);
4366 }
4367 
4368 
4369 
4370 /*!
4371  @brief Bitcount for bit block
4372 
4373  Function calculates number of 1 bits in the given array of words.
4374  Make sure the addresses are aligned.
4375 
4376  @ingroup bitfunc
4377 */
4378 inline
4380 {
4381  const bm::word_t* block_end = block + bm::set_block_size;
4382  bm::id_t count = 0;
4383 
4384 #ifdef BMVECTOPT
4385  count = VECT_BITCOUNT(block, block_end);
4386 #else
4387 #ifdef BM64OPT
4388  // 64-bit optimized algorithm. No sparse vect opt.
4389  // instead it uses 4-way parallel pipelined version
4390 
4391  const bm::id64_t* b1 = (bm::id64_t*) block;
4392  const bm::id64_t* b2 = (bm::id64_t*) block_end;
4393  do
4394  {
4395  bm::id64_t x = b1[0];
4396  bm::id64_t y = b1[1];
4397  bm::id64_t u = b1[2];
4398  bm::id64_t v = b1[3];
4399 
4400  if (x | y | u | v)
4401  {
4402  unsigned c = bitcount64_4way(x, y, u, v);
4403  BM_ASSERT(c);
4404  count += c;
4405  }
4406  b1 += 4;
4407  } while (b1 < b2);
4408 #else
4409  // For 32 bit code the fastest method is
4410  // to use bitcount table for each byte in the block.
4411  // As optimization for sparse bitsets used bits accumulator
4412  // to collect ON bits using bitwise OR.
4413  bm::word_t acc = *block++;
4414  do
4415  {
4416  bm::word_t in = *block++;
4417  bm::word_t acc_prev = acc;
4418  acc |= in;
4419  if (acc_prev &= in) // accumulator miss: counting bits
4420  {
4421  BM_INCWORD_BITCOUNT(count, acc);
4422  acc = acc_prev;
4423  }
4424  } while (block < block_end);
4425 
4426  BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4427 
4428 #endif
4429 #endif
4430  return count;
4431 }
4432 
4433 /*!
4434  @brief Bitcount for bit block
4435 
4436  Function calculates number of 1 bits in the given array of words.
4437  uses digest to understand zero areas
4438 
4439  @ingroup bitfunc
4440 */
4441 inline
4443  bm::id64_t digest) BMNOEXCEPT
4444 {
4445  bm::id_t count = 0;
4446  bm::id64_t d = digest;
4447  while (d)
4448  {
4449  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
4450 
4451  unsigned wave = bm::word_bitcount64(t - 1);
4452  unsigned off = wave * bm::set_block_digest_wave_size;
4453 
4454  const bm::bit_block_t::bunion_t* BMRESTRICT src_u =
4455  (const bm::bit_block_t::bunion_t*)(&block[off]);
4456  unsigned j = 0;
4457  do
4458  {
4459  count += bm::word_bitcount64(src_u->w64[j+0]) +
4460  bm::word_bitcount64(src_u->w64[j+1]) +
4461  bm::word_bitcount64(src_u->w64[j+2]) +
4462  bm::word_bitcount64(src_u->w64[j+3]);
4463  j += 4;
4464  } while (j < bm::set_block_digest_wave_size/2);
4465 
4466  d = bm::bmi_bslr_u64(d); // d &= d - 1;
4467  } // while
4468  return count;
4469 }
4470 
4471 
4472 
4473 /*!
4474  @brief Bitcount for bit string
4475 
4476  Added for back-compatibility purposes, not block aligned,
4477  not SIMD accelerated
4478 
4479  @ingroup bitfunc
4480 */
4481 inline
4483  const bm::word_t* block_end) BMNOEXCEPT
4484 {
4485  bm::id_t count = 0;
4486  bm::word_t acc = *block++;
4487  do
4488  {
4489  bm::word_t in = *block++;
4490  bm::word_t acc_prev = acc;
4491  acc |= in;
4492  if (acc_prev &= in) // accumulator miss: counting bits
4493  {
4494  BM_INCWORD_BITCOUNT(count, acc);
4495  acc = acc_prev;
4496  }
4497  } while (block < block_end);
4498 
4499  BM_INCWORD_BITCOUNT(count, acc); // count-in remaining accumulator
4500  return count;
4501 }
4502 
4503 
4504 
4505 /*!
4506  Function calculates number of times when bit value changed
4507  (1-0 or 0-1).
4508 
4509  For 001 result is 2
4510  010 - 3
4511  011 - 2
4512  111 - 1
4513 
4514  @ingroup bitfunc
4515 */
4516 inline
4518 {
4519  unsigned count = 1;
4520  w ^= (w >> 1);
4521 
4522  count += bm::word_bitcount(w);
4523  count -= (w >> ((sizeof(w) * 8) - 1));
4524  return count;
4525 }
4526 
4527 
4528 /*!
4529  Function calculates number of times when bit value changed
4530  @internal
4531 */
4532 inline
4533 unsigned bit_block_change32(const bm::word_t* block, unsigned size) BMNOEXCEPT
4534 {
4535  unsigned gap_count = 1;
4536 
4537  bm::word_t w, w0, w_prev, w_l;
4538  w = w0 = *block;
4539 
4540  const int w_shift = int(sizeof(w) * 8 - 1);
4541  w ^= (w >> 1);
4542  BM_INCWORD_BITCOUNT(gap_count, w);
4543  gap_count -= (w_prev = (w0 >> w_shift)); // negative value correction
4544 
4545  const bm::word_t* block_end = block + size; // bm::set_block_size;
4546  for (++block; block < block_end; ++block)
4547  {
4548  w = w0 = *block;
4549  ++gap_count;
4550  if (!w)
4551  {
4552  gap_count -= !w_prev;
4553  w_prev = 0;
4554  }
4555  else
4556  {
4557  w ^= (w >> 1);
4558  BM_INCWORD_BITCOUNT(gap_count, w);
4559 
4560  w_l = w0 & 1;
4561  gap_count -= (w0 >> w_shift); // negative value correction
4562  gap_count -= !(w_prev ^ w_l); // word border correction
4563 
4564  w_prev = (w0 >> w_shift);
4565  }
4566  } // for
4567  return gap_count;
4568 }
4569 
4570 
4571 /*!
4572  Function calculates basic bit-block statistics
4573  number of times when bit value changed (GAPS)
4574  and population count
4575  @param block - bit-block pointer
4576  @param gc - [output] gap_count
4577  @param bc - [output] bit count
4578  @internal
4579 */
4580 inline
4582  unsigned* BMRESTRICT gc, unsigned* BMRESTRICT bc) BMNOEXCEPT
4583 {
4584  BM_ASSERT(gc);
4585  BM_ASSERT(bc);
4586 
4587  #ifdef VECT_BLOCK_CHANGE_BC
4588  VECT_BLOCK_CHANGE_BC(block, gc, bc);
4589  #else
4591  *bc = bm::bit_block_count(block);
4592  #endif
4593 }
4594 
4595 
4596 
4597 /*!
4598  Function calculates number of times when bit value changed
4599  (1-0 or 0-1) in the bit block.
4600 
4601  @param block - bit-block start pointer
4602  @return number of 1-0, 0-1 transitions
4603 
4604  @ingroup bitfunc
4605 */
4606 inline
4608 {
4609 #if defined(VECT_BLOCK_CHANGE)
4610  return VECT_BLOCK_CHANGE(block, bm::set_block_size);
4611 #else
4613 #endif
4614 }
4615 
4616 /*!
4617  Check if all bits are 1 in [left, right] range
4618  @ingroup bitfunc
4619 */
4620 inline
4622  bm::word_t left,
4623  bm::word_t right) BMNOEXCEPT
4624 {
4625  BM_ASSERT(left <= right);
4626  BM_ASSERT(right <= bm::gap_max_bits-1);
4627 
4628  unsigned nword, nbit, bitcount, temp;
4629  nbit = left & bm::set_word_mask;
4630  const bm::word_t* word =
4631  block + (nword = unsigned(left >> bm::set_word_shift));
4632  if (left == right) // special case (only 1 bit to check)
4633  return (*word >> nbit) & 1u;
4634 
4635  if (nbit) // starting position is not aligned
4636  {
4637  unsigned right_margin = nbit + right - left;
4638  if (right_margin < 32)
4639  {
4640  unsigned mask =
4642  block_set_table<true>::_left[right_margin];
4643  return mask == (*word & mask);
4644  }
4645  temp = *word & block_set_table<true>::_right[nbit];
4646  if (temp != block_set_table<true>::_right[nbit])
4647  return false;
4648  bitcount = (right - left + 1u) - (32 - nbit);
4649  ++word;
4650  }
4651  else
4652  {
4653  bitcount = right - left + 1u;
4654  }
4655 
4656  // now when we are word aligned, we can scan the bit-stream
4657  const bm::id64_t maskFF64 = ~0ull;
4658  const bm::word_t maskFF = ~0u;
4659  // loop unrolled to evaluate 4 words at a time
4660  // SIMD showed no advantage, unless evaluate sub-wave intervals
4661  //
4662  for ( ;bitcount >= 128; bitcount-=128, word+=4)
4663  {
4664  bm::id64_t w64_0 = bm::id64_t(word[0]) + (bm::id64_t(word[1]) << 32);
4665  bm::id64_t w64_1 = bm::id64_t(word[2]) + (bm::id64_t(word[3]) << 32);
4666  if ((w64_0 ^ maskFF64) | (w64_1 ^ maskFF64))
4667  return false;
4668  } // for
4669 
4670  for ( ;bitcount >= 32; bitcount-=32, ++word)
4671  {
4672  if (*word != maskFF)
4673  return false;
4674  } // for
4675  BM_ASSERT(bitcount < 32);
4676 
4677  if (bitcount) // we have a tail to count
4678  {
4679  temp = *word & block_set_table<true>::_left[bitcount-1];
4680  if (temp != block_set_table<true>::_left[bitcount-1])
4681  return false;
4682  }
4683 
4684  return true;
4685 }
4686 
4687 
4688 
4689 
4690 /*!
4691  Function calculates number of 1 bits in the given array of words in
4692  the range between left anf right bits (borders included)
4693  Make sure the addr is aligned.
4694 
4695  @ingroup bitfunc
4696 */
4697 inline
4699  bm::word_t left,
4700  bm::word_t right) BMNOEXCEPT
4701 {
4702  BM_ASSERT(left <= right);
4703  BM_ASSERT(right <= bm::gap_max_bits-1);
4704 
4705  unsigned nword, nbit, bitcount, count;
4706  nbit = left & bm::set_word_mask;
4707  const bm::word_t* word =
4708  block + (nword = unsigned(left >> bm::set_word_shift));
4709  if (left == right) // special case (only 1 bit to check)
4710  {
4711  return (*word >> nbit) & 1u;
4712  }
4713 
4714  count = 0;
4715  bitcount = right - left + 1u;
4716  if (nbit) // starting position is not aligned
4717  {
4718  unsigned right_margin = nbit + right - left;
4719  if (right_margin < 32)
4720  {
4721  unsigned mask =
4723  block_set_table<true>::_left[right_margin];
4724  return bm::word_bitcount(*word & mask);
4725  }
4726  count = bm::word_bitcount(*word & block_set_table<true>::_right[nbit]);
4727  bitcount -= 32 - nbit;
4728  ++word;
4729  }
4730 
4731  // now when we are word aligned, we can count bits the usual way
4732  //
4733  #if defined(BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
4734  for ( ;bitcount >= 64; bitcount-=64)
4735  {
4736  bm::id64_t w64 = *((bm::id64_t*)word); // x86 unaligned(!) read
4737  count += unsigned(_mm_popcnt_u64(w64));
4738  word += 2;
4739  }
4740  if (bitcount >= 32)
4741  {
4742  count += bm::word_bitcount(*word++);
4743  bitcount-=32;
4744  }
4745  #else
4746  for ( ;bitcount >= 32; bitcount-=32, ++word)
4747  count += bm::word_bitcount(*word);
4748  #endif
4749  BM_ASSERT(bitcount < 32);
4750 
4751  if (bitcount) // we have a tail to count
4752  {
4753  count += bm::word_bitcount(
4754  *word & block_set_table<true>::_left[bitcount-1]);
4755  }
4756  return count;
4757 }
4758 
4759 /*!
4760  Function calculates number of 1 bits in the given array of words in
4761  the range between 0 anf right bits (borders included)
4762  Make sure the addr is aligned.
4763 
4764  @ingroup bitfunc
4765 */
4766 inline
4768  bm::word_t right) BMNOEXCEPT
4769 {
4770  BM_ASSERT(block);
4771  if (!right) // special case, first bit check
4772  return *block & 1u;
4773  bm::id_t count = 0;
4774 
4775  unsigned bitcount = right + 1;
4776 
4777  // AVX2 or 64-bit loop unroll
4778  #if defined(BMAVX2OPT) || defined(BMAVX512OPT)
4779  BM_AVX2_POPCNT_PROLOG
4780 
4781  __m256i cnt = _mm256_setzero_si256();
4782  bm::id64_t* cnt64;
4783 
4784  for ( ;bitcount >= 256; bitcount -= 256)
4785  {
4786  const __m256i* src = (__m256i*)block;
4787  __m256i xmm256 = _mm256_load_si256(src);
4788  BM_AVX2_BIT_COUNT(bc, xmm256);
4789  cnt = _mm256_add_epi64(cnt, bc);
4790 
4791  block += 8;
4792  }
4793  cnt64 = (bm::id64_t*)&cnt;
4794  count += (unsigned)(cnt64[0] + cnt64[1] + cnt64[2] + cnt64[3]);
4795  #endif
4796 
4797  for ( ;bitcount >= 64; bitcount -= 64)
4798  {
4799  bm::id64_t* p = (bm::id64_t*)block;
4800  count += bm::word_bitcount64(*p);
4801  block += 2;
4802  }
4803  if (bitcount >= 32)
4804  {
4805  count += bm::word_bitcount(*block++);
4806  bitcount-=32;
4807  }
4808 
4809  if (bitcount) // we have a tail to count
4810  {
4811  count +=
4812  bm::word_bitcount(*block & block_set_table<true>::_left[bitcount-1]);
4813  }
4814  return count;
4815 }
4816 
4817 
4818 
4819 /*!
4820  Cyclic rotation of bit-block left by 1 bit
4821  @ingroup bitfunc
4822 */
4823 inline
4825 {
4826  bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4827  for (unsigned i = 0; i < bm::set_block_size-1; ++i)
4828  {
4829  block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4830  }
4831  block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4832 }
4833 
4834 /*!
4835  @brief Unrolled cyclic rotation of bit-block left by 1 bit
4836  @param block - bit-block pointer
4837  @ingroup bitfunc
4838 */
4839 inline
4841 {
4842  bm::word_t co_flag = (block[0] >> 31) & 1; // carry over bit
4843  const unsigned unroll_factor = 4;
4844  bm::word_t w0, w1, w2, w3;
4845 
4846  unsigned i;
4847  for (i = 0; i < bm::set_block_size - unroll_factor; i += unroll_factor)
4848  {
4849  w0 = block[i + 1] >> 31;
4850  w1 = block[i + 2] >> 31;
4851  w2 = block[i + 3] >> 31;
4852  w3 = block[i + 4] >> 31;
4853 
4854  block[0 + i] = (block[0 + i] << 1) | w0;
4855  block[1 + i] = (block[1 + i] << 1) | w1;
4856  block[2 + i] = (block[2 + i] << 1) | w2;
4857  block[3 + i] = (block[3 + i] << 1) | w3;
4858  }
4859  block[i] = (block[i] << 1) | (block[i + 1] >> 31);
4860  block[i + 1] = (block[i + 1] << 1) | (block[i + 2] >> 31);
4861  block[i + 2] = (block[i + 2] << 1) | (block[i + 3] >> 31);
4862  block[set_block_size - 1] = (block[set_block_size - 1] << 1) | co_flag;
4863 }
4864 
4865 /*!
4866  @brief insert bit into position and shift the rest right with carryover
4867 
4868  @param block - bit-block pointer
4869  @param bitpos - bit position to insert
4870  @param value - bit value (0|1) to insert
4871 
4872  @return carry over value
4873  @ingroup bitfunc
4874 */
4875 inline
4877  unsigned bitpos, bool value) BMNOEXCEPT
4878 {
4879  BM_ASSERT(block);
4880  BM_ASSERT(bitpos < 65536);
4881 
4882  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
4883  unsigned nword = unsigned(nbit >> bm::set_word_shift);
4884  nbit &= bm::set_word_mask;
4885 
4886  bm::word_t co_flag = 0;
4887  if (nbit)
4888  {
4889  unsigned r_mask = bm::block_set_table<true>::_right[nbit];
4890  bm::word_t w = block[nword] & r_mask;
4891  bm::word_t wl = block[nword] & ~r_mask;
4892  co_flag = w >> 31;
4893  w <<= 1u;
4894  block[nword] = w | (unsigned(value) << nbit) | wl;
4895  ++nword;
4896  }
4897  else
4898  {
4899  co_flag = value;
4900  }
4901 
4902  for (unsigned i = nword; i < bm::set_block_size; ++i)
4903  {
4904  bm::word_t w = block[i];
4905  bm::word_t co_flag1 = w >> 31;
4906  w = (w << 1u) | co_flag;
4907  block[i] = w;
4908  co_flag = co_flag1;
4909  } // for i
4910  return co_flag;
4911 }
4912 
4913 
4914 
4915 /*!
4916  @brief Right bit-shift bitblock by 1 bit (reference)
4917  @param block - bit-block pointer
4918  @param empty_acc - [out] contains 0 if block is empty
4919  @param co_flag - carry over from the previous block
4920 
4921  @return carry over bit (1 or 0)
4922  @ingroup bitfunc
4923 */
4924 inline
4926  bm::word_t* BMRESTRICT empty_acc,
4927  bm::word_t co_flag) BMNOEXCEPT
4928 {
4929  BM_ASSERT(block);
4930  BM_ASSERT(empty_acc);
4931 
4932  bm::word_t acc = 0;
4933  for (unsigned i = 0; i < bm::set_block_size; ++i)
4934  {
4935  bm::word_t w = block[i];
4936  bm::word_t co_flag1 = w >> 31;
4937  acc |= w = (w << 1u) | co_flag;
4938  block[i] = w;
4939  co_flag = co_flag1;
4940  }
4941  *empty_acc = acc;
4942  return co_flag;
4943 }
4944 
4945 /*!
4946  @brief Right bit-shift of bit-block by 1 bit (loop unrolled)
4947  @param block - bit-block pointer
4948  @param empty_acc - [out] contains 0 if block is empty
4949  @param co_flag - carry over from the previous block
4950 
4951  @return carry over bit (1 or 0)
4952  @ingroup bitfunc
4953 */
4954 inline
4956  bm::word_t* BMRESTRICT empty_acc,
4957  bm::word_t co_flag) BMNOEXCEPT
4958 {
4959  BM_ASSERT(block);
4960  BM_ASSERT(empty_acc);
4961  #if defined(VECT_SHIFT_R1)
4962  return VECT_SHIFT_R1(block, empty_acc, co_flag);
4963  #else
4964  return bm::bit_block_shift_r1(block, empty_acc, co_flag);
4965  #endif
4966 }
4967 
4968 
4969 /*!
4970  @brief Left bit-shift bitblock by 1 bit (reference)
4971  @param block - bit-block pointer
4972  @param empty_acc - [out] contains 0 if block is empty
4973  @param co_flag - carry over from the prev/next block
4974 
4975  @return carry over bit (1 or 0)
4976 
4977  @ingroup bitfunc
4978 */
4979 inline
4981  bm::word_t* empty_acc, bm::word_t co_flag) BMNOEXCEPT
4982 {
4983  BM_ASSERT(block);
4984  BM_ASSERT(empty_acc);
4985 
4986  bm::word_t acc = 0;
4987  for (int i = bm::set_block_size-1; i >= 0; --i)
4988  {
4989  bm::word_t w = block[i];
4990  bm::word_t co_flag1 = w & 1u;
4991  acc |= w = (w >> 1u) | (co_flag << 31u);
4992  block[i] = w;
4993  co_flag = co_flag1;
4994  }
4995 
4996  *empty_acc = acc;
4997  return co_flag;
4998 }
4999 
5000 /*!
5001  @brief Left bit-shift of bit-block by 1 bit (loop unrolled)
5002  @param block - bit-block pointer
5003  @param empty_acc - [out] contains 0 if block is empty
5004  @param co_flag - carry over from the prev/next block
5005 
5006  @return carry over bit (1 or 0)
5007  @ingroup bitfunc
5008 */
5009 inline
5011  bm::word_t* empty_acc,
5012  bm::word_t co_flag) BMNOEXCEPT
5013 {
5014  BM_ASSERT(block);
5015  BM_ASSERT(empty_acc);
5016  #if defined(VECT_SHIFT_L1)
5017  return VECT_SHIFT_L1(block, empty_acc, co_flag);
5018  #else
5019  return bm::bit_block_shift_l1(block, empty_acc, co_flag);
5020  #endif
5021 }
5022 
5023 /*!
5024  @brief erase bit from position and shift the rest right with carryover
5025 
5026  @param block - bit-block pointer
5027  @param bitpos - bit position to insert
5028  @param carry_over - bit value to add to the end (0|1)
5029 
5030  @ingroup bitfunc
5031 */
5032 inline
5034  unsigned bitpos,
5035  bool carry_over) BMNOEXCEPT
5036 {
5037  BM_ASSERT(block);
5038  BM_ASSERT(bitpos < 65536);
5039 
5040  if (!bitpos)
5041  {
5042  bm::word_t acc;
5043  bm::bit_block_shift_l1_unr(block, &acc, carry_over);
5044  return;
5045  }
5046 
5047  unsigned nbit = unsigned(bitpos & bm::set_block_mask);
5048  unsigned nword = unsigned(nbit >> bm::set_word_shift);
5049  nbit &= bm::set_word_mask;
5050 
5051  bm::word_t co_flag = carry_over;
5052  for (unsigned i = bm::set_block_size-1; i > nword; --i)
5053  {
5054  bm::word_t w = block[i];
5055  bm::word_t co_flag1 = w & 1u;
5056  w = (w >> 1u) | (co_flag << 31u);
5057  block[i] = w;
5058  co_flag = co_flag1;
5059  }
5060 
5061  if (nbit)
5062  {
5063  unsigned r_mask = bm::block_set_table<true>::_right[nbit];
5064  bm::word_t w = block[nword] & r_mask;
5065  bm::word_t wl = block[nword] & ~r_mask;
5066  w &= ~(1 << nbit); // clear the removed bit
5067  w >>= 1u;
5068  w |= wl | (co_flag << 31u);
5069  block[nword] = w;
5070  }
5071  else
5072  {
5073  block[nword] = (block[nword] >> 1u) | (co_flag << 31u);
5074  }
5075 }
5076 
5077 
5078 /*!
5079  @brief Right bit-shift of bit-block by 1 bit (reference) + AND
5080  @param block - bit-block pointer
5081  @param co_flag - carry over from the previous block
5082  @param mask_block - mask bit-block pointer
5083  @param digest - block digest
5084 
5085  @return carry over bit (1 or 0)
5086  @ingroup bitfunc
5087 */
5088 inline
5090  bm::word_t co_flag,
5091  const bm::word_t* BMRESTRICT mask_block,
5093 {
5094  BM_ASSERT(block);
5095  BM_ASSERT(mask_block);
5096  BM_ASSERT(digest && *digest);
5097 
5098 
5099  bm::id64_t d = *digest;
5100 
5101  unsigned di = 0;
5102  if (!co_flag)
5103  {
5104  bm::id64_t t = d & -d;
5105  di = bm::word_bitcount64(t - 1); // find start bit-index
5106  }
5107 
5108  for (; di < 64; ++di)
5109  {
5110  const unsigned d_base = di * bm::set_block_digest_wave_size;
5111  bm::id64_t dmask = (1ull << di);
5112  if (d & dmask) // digest stride not empty
5113  {
5114  bm::word_t acc = 0;
5115  for (unsigned i = d_base; i < d_base + bm::set_block_digest_wave_size; ++i)
5116  {
5118 
5119  bm::word_t w = block[i];
5120  bm::word_t co_flag1 = w >> 31;
5121  w = (w << 1u) | co_flag;
5122  acc |= block[i] = w & mask_block[i];
5123  co_flag = co_flag1;
5124  }
5125  if (!acc)
5126  d &= ~dmask; // update digest: clear stride bit
5127  }
5128  else // stride is empty
5129  {
5130  BM_ASSERT(block[d_base + bm::set_block_digest_wave_size -1]==0);
5131  BM_ASSERT(block[d_base]==0);
5132 
5133  if (co_flag) // there is carry-over
5134  {
5135  BM_ASSERT(co_flag == 1);
5136  BM_ASSERT(block[d_base] == 0);
5137 
5138  block[d_base] = co_flag & mask_block[d_base];
5139  if (block[d_base])
5140  d |= dmask; // update digest
5141  co_flag = 0;
5142  }
5143  }
5144  } // for di
5145 
5146  *digest = d;
5147  return co_flag;
5148 }
5149 
5150 /*!
5151  @brief Right bit-shift bitblock by 1 bit (reference) + AND
5152  @param block - bit-block pointer
5153  @param co_flag - carry over from the previous block
5154  @param mask_block - mask bit-block pointer
5155  @param digest - block digest
5156 
5157  @return carry over bit (1 or 0)
5158  @ingroup bitfunc
5159 */
5160 inline
5162  bm::word_t co_flag,
5163  const bm::word_t* BMRESTRICT mask_block,
5165 {
5166  BM_ASSERT(block);
5167  BM_ASSERT(mask_block);
5168  BM_ASSERT(digest);
5169 
5170  #if defined(VECT_SHIFT_R1_AND)
5171  return VECT_SHIFT_R1_AND(block, co_flag, mask_block, digest);
5172  #else
5173  return bm::bit_block_shift_r1_and(block, co_flag, mask_block, digest);
5174  #endif
5175 }
5176 
5177 
5178 /*!
5179  Function calculates if there is any number of 1 bits
5180  in the given array of words in the range between left anf right bits
5181  (borders included). Make sure the addresses are aligned.
5182 
5183  @ingroup bitfunc
5184 */
5185 inline
5187  bm::word_t left,
5188  bm::word_t right) BMNOEXCEPT
5189 {
5190  BM_ASSERT(left <= right);
5191 
5192  unsigned nbit = left; // unsigned(left & bm::set_block_mask);
5193  unsigned nword = unsigned(nbit >> bm::set_word_shift);
5194  nbit &= bm::set_word_mask;
5195 
5196  const bm::word_t* word = block + nword;
5197 
5198  if (left == right) // special case (only 1 bit to check)
5199  {
5200  return (*word >> nbit) & 1;
5201  }
5202  unsigned acc;
5203  unsigned bitcount = right - left + 1;
5204 
5205  if (nbit) // starting position is not aligned
5206  {
5207  unsigned right_margin = nbit + (right - left);
5208  if (right_margin < 32)
5209  {
5210  unsigned mask =
5212  block_set_table<true>::_left[right_margin];
5213  return *word & mask;
5214  }
5215  else
5216  {
5217  acc = *word & block_set_table<true>::_right[nbit];
5218  if (acc)
5219  return acc;
5220  bitcount -= 32 - nbit;
5221  }
5222  ++word;
5223  }
5224 
5225  // loop unrolled to evaluate 4 words at a time
5226  // SIMD showed no advantage, unless evaluate sub-wave intervals
5227  //
5228  for ( ;bitcount >= 128; bitcount-=128, word+=4)
5229  {
5230  acc = word[0] | word[1] | word[2] | word[3];
5231  if (acc)
5232  return acc;
5233  } // for
5234 
5235  acc = 0;
5236  for ( ;bitcount >= 32; bitcount -= 32)
5237  {
5238  acc |= *word++;
5239  } // for
5240 
5241  if (bitcount) // we have a tail to count
5242  acc |= (*word) & block_set_table<true>::_left[bitcount-1];
5243 
5244  return acc;
5245 }
5246 
5247 // ----------------------------------------------------------------------
5248 
5249 /*! Function inverts block of bits
5250  @ingroup bitfunc
5251 */
5252 template<typename T>
5253 void bit_invert(T* start) BMNOEXCEPT
5254 {
5256 #ifdef BMVECTOPT
5257  VECT_INVERT_BLOCK(start);
5258 #else
5259  T* end = (T*)((unsigned*)(start) + bm::set_block_size);
5260  do
5261  {
5262  start[0] = ~start[0];
5263  start[1] = ~start[1];
5264  start[2] = ~start[2];
5265  start[3] = ~start[3];
5266  start+=4;
5267  } while (start < end);
5268 #endif
5269 }
5270 
5271 // ----------------------------------------------------------------------
5272 
5273 /*! @brief Returns "true" if all bits in the block are 1
5274  @ingroup bitfunc
5275 */
5276 inline
5278 {
5279 #if defined(BMSSE42OPT) || defined(BMAVX2OPT)
5280  return VECT_IS_ONE_BLOCK(start);
5281 #else
5282  const bm::word_t* BMRESTRICT src_end = (bm::word_t*)start + bm::set_block_size;
5283  const bm::wordop_t* end = (const bm::wordop_t*)src_end;
5284  do
5285  {
5286  bm::wordop_t tmp =
5287  start[0] & start[1] & start[2] & start[3];
5288  if (tmp != bm::all_bits_mask)
5289  return false;
5290  start += 4;
5291  } while (start < end);
5292  return true;
5293 #endif
5294 }
5295 
5296 // ----------------------------------------------------------------------
5297 
5298 /*! @brief Returns "true" if all bits are 1 in the block [left, right]
5299  Function check for block varieties
5300  @internal
5301 */
5302 inline
5304  unsigned left, unsigned right) BMNOEXCEPT
5305 {
5306  BM_ASSERT(left <= right);
5307  BM_ASSERT(right < bm::gap_max_bits);
5308  if (block)
5309  {
5310  if (BM_IS_GAP(block))
5311  return bm::gap_is_all_one_range(BMGAP_PTR(block), left, right);
5312  if (block == FULL_BLOCK_FAKE_ADDR)
5313  return true;
5314  return bm::bit_block_is_all_one_range(block, left, right);
5315  }
5316  return false;
5317 }
5318 
5319 /*! @brief Returns "true" if all bits are 1 in the block [left, right]
5320  and border bits are 0
5321  @internal
5322 */
5323 inline
5324 bool block_is_interval(const bm::word_t* const BMRESTRICT block,
5325  unsigned left, unsigned right) BMNOEXCEPT
5326 {
5327  BM_ASSERT(left <= right);
5328  BM_ASSERT(right < bm::gap_max_bits-1);
5329 
5330  if (block)
5331  {
5332  bool is_left, is_right, all_one;
5333  if (BM_IS_GAP(block))
5334  {
5335  const bm::gap_word_t* gap = BMGAP_PTR(block);
5336  all_one = bm::gap_is_interval(gap, left, right);
5337  return all_one;
5338  }
5339  else // bit-block
5340  {
5341  if (block == FULL_BLOCK_FAKE_ADDR)
5342  return false;
5343  unsigned nword = ((left-1) >> bm::set_word_shift);
5344  is_left = block[nword] & (1u << ((left-1) & bm::set_word_mask));
5345  if (is_left == false)
5346  {
5347  nword = ((right + 1) >> bm::set_word_shift);
5348  is_right = block[nword] & (1u << ((right + 1) & bm::set_word_mask));
5349  if (is_right == false)
5350  {
5351  all_one = bm::bit_block_is_all_one_range(block, left, right);
5352  return all_one;
5353  }
5354  }
5355  }
5356  }
5357 
5358  return false;
5359 }
5360 
5361 // ----------------------------------------------------------------------
5362 
5363 /**
5364  \brief Searches for the last 1 bit in the 111 interval of a BIT block
5365  \param block - BIT buffer
5366  \param nbit - bit index to start checking from
5367  \param pos - [out] found value
5368 
5369  \return false if not found
5370  @ingroup bitfunc
5371 */
5372 inline
5374  unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5375 {
5376  BM_ASSERT(block);
5377  BM_ASSERT(pos);
5378 
5379  unsigned nword = unsigned(nbit >> bm::set_word_shift);
5380  unsigned bit_pos = (nbit & bm::set_word_mask);
5381  bm::word_t w = block[nword];
5382  w &= (1u << bit_pos);
5383  if (!w)
5384  return false;
5385 
5386  if (nbit == bm::gap_max_bits-1)
5387  {
5388  *pos = bm::gap_max_bits-1;
5389  return true;
5390  }
5391  *pos = nbit;
5392 
5393  ++nbit;
5394  nword = unsigned(nbit >> bm::set_word_shift);
5395  bit_pos = (nbit & bm::set_word_mask);
5396 
5397  w = (~block[nword]) >> bit_pos;
5398  w <<= bit_pos; // clear the trailing bits
5399  if (w)
5400  {
5401  bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5402  *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5403  return true;
5404  }
5405 
5406  for (++nword; nword < bm::set_block_size; ++nword)
5407  {
5408  w = ~block[nword];
5409  if (w)
5410  {
5411  bit_pos = bm::bit_scan_forward32(w); // trailing zeros
5412  *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))-1);
5413  return true;
5414  }
5415  } // for nword
5416 
5417  // 0 not found, all block is 1s...
5418  *pos = bm::gap_max_bits-1;
5419  return true;
5420 }
5421 
5422 
5423 /*! @brief Find end of the current 111 interval
5424  @return search result code 0 - not found, 1 found, 2 - found at the end
5425  @internal
5426 */
5427 inline
5429  unsigned nbit_from,
5430  unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5431 {
5432  BM_ASSERT(block && found_nbit);
5433  BM_ASSERT(nbit_from < bm::gap_max_bits);
5434 
5435  bool b;
5436  if (BM_IS_GAP(block))
5437  {
5438  const bm::gap_word_t* gap = BMGAP_PTR(block);
5439  b = bm::gap_find_interval_end(gap, nbit_from, found_nbit);
5440  if (b && *found_nbit == bm::gap_max_bits-1)
5441  return 2; // end of block, keep searching
5442  }
5443  else // bit-block
5444  {
5445  if (IS_FULL_BLOCK(block))
5446  {
5447  *found_nbit = bm::gap_max_bits-1;
5448  return 2;
5449  }
5450  b = bm::bit_block_find_interval_end(block, nbit_from, found_nbit);
5451  if (b && *found_nbit == bm::gap_max_bits-1)
5452  return 2; // end of block, keep searching
5453  }
5454  return b;
5455 }
5456 
5457 // ----------------------------------------------------------------------
5458 
5459 /**
5460  \brief Searches for the first 1 bit in the 111 interval of a BIT block
5461  \param block - BIT buffer
5462  \param nbit - bit index to start checking from
5463  \param pos - [out] found value
5464 
5465  \return false if not found
5466  @ingroup bitfunc
5467 */
5468 inline
5470  unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
5471 {
5472  BM_ASSERT(block);
5473  BM_ASSERT(pos);
5474 
5475  unsigned nword = unsigned(nbit >> bm::set_word_shift);
5476  unsigned bit_pos = (nbit & bm::set_word_mask);
5477  bm::word_t w = block[nword];
5478  w &= (1u << bit_pos);
5479  if (!w)
5480  return false;
5481 
5482  if (nbit == 0)
5483  {
5484  *pos = 0;
5485  return true;
5486  }
5487  *pos = nbit;
5488 
5489  --nbit;
5490  nword = unsigned(nbit >> bm::set_word_shift);
5491  bit_pos = (nbit & bm::set_word_mask);
5492 
5493  w = (~block[nword]) & block_set_table<true>::_left[bit_pos];
5494  if (w)
5495  {
5496  bit_pos = bm::bit_scan_reverse32(w);
5497  *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5498  return true;
5499  }
5500 
5501  if (nword)
5502  {
5503  for (--nword; true; --nword)
5504  {
5505  w = ~block[nword];
5506  if (w)
5507  {
5508  bit_pos = bm::bit_scan_reverse32(w); // trailing zeros
5509  *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t)))+1);
5510  return true;
5511  }
5512  if (!nword)
5513  break;
5514  } // for nword
5515  }
5516 
5517  // 0 not found, all block is 1s...
5518  *pos = 0;
5519  return true;
5520 }
5521 
5522 
5523 /*! @brief Find start of the current 111 interval
5524  @return search result code 0 - not found, 1 found, 2 - found at the start
5525  @internal
5526 */
5527 inline
5529  unsigned nbit_from,
5530  unsigned* BMRESTRICT found_nbit) BMNOEXCEPT
5531 {
5532  BM_ASSERT(block && found_nbit);
5533  BM_ASSERT(nbit_from < bm::gap_max_bits);
5534  bool b;
5535  if (BM_IS_GAP(block))
5536  {
5537  const bm::gap_word_t* gap = BMGAP_PTR(block);
5538  b = bm::gap_find_interval_start(gap, nbit_from, found_nbit);
5539  if (b && *found_nbit == 0)
5540  return 2; // start of block, keep searching
5541  }
5542  else // bit-block
5543  {
5544  if (IS_FULL_BLOCK(block))
5545  {
5546  *found_nbit = 0;
5547  return 2;
5548  }
5549  b = bm::bit_block_find_interval_start(block, nbit_from, found_nbit);
5550  if (b && *found_nbit == 0)
5551  return 2; // start of block, keep searching
5552  }
5553  return b;
5554 }
5555 
5556 // ----------------------------------------------------------------------
5557 
5558 /*! @brief Returns "true" if one bit is set in the block [left, right]
5559  Function check for block varieties
5560  @internal
5561 */
5562 inline
5563 bool block_any_range(const bm::word_t* const BMRESTRICT block,
5564  unsigned left, unsigned right) BMNOEXCEPT
5565 {
5566  BM_ASSERT(left <= right);
5567  BM_ASSERT(right < bm::gap_max_bits);
5568  if (!block)
5569  return false;
5570  if (BM_IS_GAP(block))
5571  return bm::gap_any_range(BMGAP_PTR(block), left, right);
5572  if (IS_FULL_BLOCK(block))
5573  return true;
5574  return bm::bit_block_any_range(block, left, right);
5575 }
5576 
5577 // ----------------------------------------------------------------------
5578 
5579 /*! @brief Returns "true" if one bit is set in the block
5580  Function check for block varieties
5581  @internal
5582 */
5583 inline
5584 bool block_any(const bm::word_t* const BMRESTRICT block) BMNOEXCEPT
5585 {
5586  if (!block)
5587  return false;
5588  if (IS_FULL_BLOCK(block))
5589  return true;
5590  bool all_zero = (BM_IS_GAP(block)) ?
5592  : bm::bit_is_all_zero(block);
5593  return !all_zero;
5594 }
5595 
5596 
5597 
5598 // ----------------------------------------------------------------------
5599 
5600 // GAP blocks manipulation functions:
5601 
5602 /*! \brief GAP and functor */
5603 BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5604 {
5605  return v1 & v2;
5606 }
5607 
5608 
5609 /*! \brief GAP xor functor */
5610 BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5611 {
5612  return v1 ^ v2;
5613 }
5614 
5615 
5616 /*! \brief GAP or functor */
5617 BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5618 {
5619  return v1 | v2;
5620 }
5621 
5622 /*! \brief GAP or functor */
5623 BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
5624 {
5625  return v1 & ~v2;
5626 }
5627 
5628 
5629 /*!
5630  \brief GAP AND operation.
5631 
5632  Function performs AND logical operation on gap vectors.
5633  If possible function put the result into vect1 and returns this
5634  pointer. Otherwise result is put into tmp_buf, which should be
5635  twice of the vector size.
5636 
5637  \param vect1 - operand 1
5638  \param vect2 - operand 2
5639  \param tmp_buf - pointer on temporary buffer
5640  \param dsize - out size of the destination
5641  \return Result pointer (tmp_buf OR vect1)
5642 
5643  @ingroup gapfunc
5644 */
5647  const gap_word_t* BMRESTRICT vect2,
5648  gap_word_t* BMRESTRICT tmp_buf,
5649  unsigned& dsize) BMNOEXCEPT
5650 {
5651  bm::gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::and_op, dsize);
5652  return tmp_buf;
5653 }
5654 
5655 /*!
5656  \brief GAP AND operation test.
5657 
5658  Function performs AND logical operation on gap vectors.
5659  If possible function put the result into vect1 and returns this
5660  pointer. Otherwise result is put into tmp_buf, which should be
5661  twice of the vector size.
5662 
5663  \param vect1 - operand 1
5664  \param vect2 - operand 2
5665  \return non zero value if operation returns any 1 bit
5666 
5667  @ingroup gapfunc
5668 */
5671  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5672 {
5673  return gap_buff_any_op(vect1, 0, vect2, 0, and_op);
5674 }
5675 
5676 
5677 /*!
5678  \brief GAP bitcount AND operation test.
5679 
5680  \param vect1 - operand 1
5681  \param vect2 - operand 2
5682  \return bitcount of vect1 AND vect2
5683 
5684  @ingroup gapfunc
5685 */
5686 inline
5687 unsigned gap_count_and(const gap_word_t* BMRESTRICT vect1,
5688  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5689 {
5690  return gap_buff_count_op(vect1, vect2, and_op);
5691 }
5692 
5693 
5694 
5695 /*!
5696  \brief GAP XOR operation.
5697 
5698  Function performs XOR logical operation on gap vectors.
5699  If possible function put the result into vect1 and returns this
5700  pointer. Otherwise result is put into tmp_buf, which should be
5701  twice of the vector size.
5702 
5703  \param vect1 - operand 1
5704  \param vect2 - operand 2
5705  \param tmp_buf - pointer on temporary buffer
5706  \param dsize - out destination size
5707  \return Result pointer (tmp_buf)
5708 
5709  @ingroup gapfunc
5710 */
5711 inline
5713  const gap_word_t* BMRESTRICT vect2,
5714  gap_word_t* BMRESTRICT tmp_buf,
5715  unsigned& dsize) BMNOEXCEPT
5716 {
5717  gap_buff_op(tmp_buf, vect1, 0, vect2, 0, bm::xor_op, dsize);
5718  return tmp_buf;
5719 }
5720 
5721 /*! Light weight gap_operation_xor for len prediction
5722  @ingroup gapfunc
5723 */
5724 inline
5726  const gap_word_t* BMRESTRICT vect2,
5727  unsigned& dsize,
5728  unsigned limit) BMNOEXCEPT
5729 {
5730  return gap_buff_dry_op(vect1, vect2, bm::xor_op, dsize, limit);
5731 }
5732 
5733 
5734 /*!
5735  \brief GAP XOR operation test.
5736 
5737  Function performs AND logical operation on gap vectors.
5738  If possible function put the result into vect1 and returns this
5739  pointer. Otherwise result is put into tmp_buf, which should be
5740  twice of the vector size.
5741 
5742  \param vect1 - operand 1
5743  \param vect2 - operand 2
5744  \return non zero value if operation returns any 1 bit
5745 
5746  @ingroup gapfunc
5747 */
5750  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5751 {
5752  return gap_buff_any_op(vect1, 0, vect2, 0, bm::xor_op);
5753 }
5754 
5755 /*!
5756  \brief GAP bitcount XOR operation test.
5757 
5758  \param vect1 - operand 1
5759  \param vect2 - operand 2
5760  \return bitcount of vect1 XOR vect2
5761 
5762  @ingroup gapfunc
5763 */
5765 unsigned gap_count_xor(const gap_word_t* BMRESTRICT vect1,
5766  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5767 {
5768  return gap_buff_count_op(vect1, vect2, bm::xor_op);
5769 }
5770 
5771 
5772 /*!
5773  \brief GAP OR operation.
5774 
5775  Function performs OR logical oparation on gap vectors.
5776  If possible function put the result into vect1 and returns this
5777  pointer. Otherwise result is put into tmp_buf, which should be
5778  twice of the vector size.
5779 
5780  \param vect1 - operand 1
5781  \param vect2 - operand 2
5782  \param tmp_buf - pointer on temporary buffer
5783  \param dsize - out destination size
5784 
5785  \return Result pointer (tmp_buf)
5786 
5787  @ingroup gapfunc
5788 */
5789 inline
5791  const gap_word_t* BMRESTRICT vect2,
5792  gap_word_t* BMRESTRICT tmp_buf,
5793  unsigned& dsize) BMNOEXCEPT
5794 {
5795  gap_buff_op(tmp_buf, vect1, 1, vect2, 1, bm::and_op, dsize);
5796  gap_invert(tmp_buf);
5797  return tmp_buf;
5798 }
5799 
5800 /*!
5801  \brief GAP bitcount OR operation test.
5802 
5803  \param vect1 - operand 1
5804  \param vect2 - operand 2
5805  \return bitcount of vect1 OR vect2
5806 
5807  @ingroup gapfunc
5808 */
5810 unsigned gap_count_or(const gap_word_t* BMRESTRICT vect1,
5811  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5812 {
5813  return gap_buff_count_op(vect1, vect2, bm::or_op);
5814 }
5815 
5816 
5817 
5818 /*!
5819  \brief GAP SUB (AND NOT) operation.
5820 
5821  Function performs SUB logical operation on gap vectors.
5822  If possible function put the result into vect1 and returns this
5823  pointer. Otherwise result is put into tmp_buf, which should be
5824  twice of the vector size.
5825 
5826  \param vect1 - operand 1
5827  \param vect2 - operand 2
5828  \param tmp_buf - pointer on temporary buffer
5829  \param dsize - out destination size
5830 
5831  \return Result pointer (tmp_buf)
5832 
5833  @ingroup gapfunc
5834 */
5836  const gap_word_t* BMRESTRICT vect2,
5837  gap_word_t* BMRESTRICT tmp_buf,
5838  unsigned& dsize) BMNOEXCEPT
5839 {
5840  gap_buff_op(tmp_buf, vect1, 0, vect2, 1, and_op, dsize);
5841  return tmp_buf;
5842 }
5843 
5844 
5845 /*!
5846  \brief GAP SUB operation test.
5847 
5848  Function performs AND logical operation on gap vectors.
5849  If possible function put the result into vect1 and returns this
5850  pointer. Otherwise result is put into tmp_buf, which should be
5851  twice of the vector size.
5852 
5853  \param vect1 - operand 1
5854  \param vect2 - operand 2
5855  \return non zero value if operation returns any 1 bit
5856 
5857  @ingroup gapfunc
5858 */
5861  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5862 {
5863  return gap_buff_any_op(vect1, 0, vect2, 1, bm::and_op);
5864 }
5865 
5866 
5867 /*!
5868 \brief GAP bitcount SUB (AND NOT) operation test.
5869 
5870 \param vect1 - operand 1
5871 \param vect2 - operand 2
5872 \return bitcount of vect1 SUB (AND NOT) vect2
5873 
5874 @ingroup gapfunc
5875 */
5877 unsigned gap_count_sub(const gap_word_t* BMRESTRICT vect1,
5878  const gap_word_t* BMRESTRICT vect2) BMNOEXCEPT
5879 {
5880  return gap_buff_count_op(vect1, vect2, bm::sub_op);
5881 }
5882 
5883 
5884 // ----------------------------------------------------------------------
5885 
5886 // BIT blocks manipulation functions:
5887 
5888 
5889 /*!
5890  \brief Bitblock copy operation.
5891 
5892  \param dst - destination block.
5893  \param src - source block.
5894 
5895  @ingroup bitfunc
5896 */
5897 inline
5899  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5900 {
5901 #ifdef BMVECTOPT
5902  VECT_COPY_BLOCK(dst, src);
5903 #else
5904  ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5905 #endif
5906 }
5907 
5908 /*!
5909  \brief Bitblock copy/stream operation.
5910 
5911  \param dst - destination block.
5912  \param src - source block.
5913 
5914  @ingroup bitfunc
5915 */
5916 inline
5918  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5919 {
5920 #ifdef VECT_STREAM_BLOCK
5921  VECT_STREAM_BLOCK(dst, src);
5922 #else
5923  ::memcpy(dst, src, bm::set_block_size * sizeof(bm::word_t));
5924 #endif
5925 }
5926 
5927 
5928 /*!
5929  \brief Plain bitblock AND operation.
5930  Function does not analyse availability of source and destination blocks.
5931 
5932  \param dst - destination block.
5933  \param src - source block.
5934 
5935  \return 0 if AND operation did not produce anything (no 1s in the output)
5936 
5937  @ingroup bitfunc
5938 */
5939 inline
5941  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
5942 {
5943  BM_ASSERT(dst);
5944  BM_ASSERT(src);
5945  BM_ASSERT(dst != src);
5946 
5947 #ifdef BMVECTOPT
5948  bm::id64_t acc = VECT_AND_BLOCK(dst, src);
5949 #else
5950  unsigned arr_sz = bm::set_block_size / 2;
5951 
5954 
5955  bm::id64_t acc = 0;
5956  for (unsigned i = 0; i < arr_sz; i+=4)
5957  {
5958  acc |= dst_u->w64[i] &= src_u->w64[i];
5959  acc |= dst_u->w64[i+1] &= src_u->w64[i+1];
5960  acc |= dst_u->w64[i+2] &= src_u->w64[i+2];
5961  acc |= dst_u->w64[i+3] &= src_u->w64[i+3];
5962  }
5963 #endif
5964  return acc;
5965 }
5966 
5967 /*!
5968  \brief digest based bit-block AND
5969 
5970  \param dst - destination block.
5971  \param src - source block.
5972  \param digest - known digest of dst block
5973 
5974  \return new digest
5975 
5976  @ingroup bitfunc
5977 */
5978 inline
5980  const bm::word_t* BMRESTRICT src,
5981  bm::id64_t digest) BMNOEXCEPT
5982 {
5983  BM_ASSERT(dst);
5984  BM_ASSERT(src);
5985  BM_ASSERT(dst != src);
5986 
5987  const bm::id64_t mask(1ull);
5988  bm::id64_t d = digest;
5989  while (d)
5990  {
5991  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
5992 
5993  unsigned wave = bm::word_bitcount64(t - 1);
5994  unsigned off = wave * bm::set_block_digest_wave_size;
5995 
5996  #if defined(VECT_AND_DIGEST)
5997  bool all_zero = VECT_AND_DIGEST(&dst[off], &src[off]);
5998  if (all_zero)
5999  digest &= ~(mask << wave);
6000  #else
6001  const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
6003 
6004  bm::id64_t acc = 0;
6005  unsigned j = 0;
6006  do
6007  {
6008  acc |= dst_u->w64[j+0] &= src_u->w64[j+0];
6009  acc |= dst_u->w64[j+1] &= src_u->w64[j+1];
6010  acc |= dst_u->w64[j+2] &= src_u->w64[j+2];
6011  acc |= dst_u->w64[j+3] &= src_u->w64[j+3];
6012  j+=4;
6013  } while (j < bm::set_block_digest_wave_size/2);
6014 
6015  if (!acc) // all zero
6016  digest &= ~(mask << wave);
6017  #endif
6018 
6019  d = bm::bmi_bslr_u64(d); // d &= d - 1;
6020  } // while
6021 
6022  return digest;
6023 }
6024 
6025 
6026 /*!
6027  \brief digest based bit-block AND 5-way
6028 
6029  \return new digest
6030 
6031  @ingroup bitfunc
6032 */
6033 inline
6035  const bm::word_t* BMRESTRICT src0,
6036  const bm::word_t* BMRESTRICT src1,
6037  const bm::word_t* BMRESTRICT src2,
6038  const bm::word_t* BMRESTRICT src3,
6039  bm::id64_t digest) BMNOEXCEPT
6040 {
6041  BM_ASSERT(dst);
6042  BM_ASSERT(src0 && src1 && src2 && src3);
6043 
6044  const bm::id64_t mask(1ull);
6045  bm::id64_t d = digest;
6046  while (d)
6047  {
6048  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6049 
6050  unsigned wave = bm::word_bitcount64(t - 1);
6051  unsigned off = wave * bm::set_block_digest_wave_size;
6052 
6053 #if defined(VECT_AND_DIGEST_5WAY)
6054  bool all_zero = VECT_AND_DIGEST_5WAY(&dst[off], &src0[off], &src1[off], &src2[off], &src3[off]);
6055  if (all_zero)
6056  digest &= ~(mask << wave);
6057 #else
6058  const bm::bit_block_t::bunion_t* BMRESTRICT src_u0 = (const bm::bit_block_t::bunion_t*)(&src0[off]);
6059  const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 = (const bm::bit_block_t::bunion_t*)(&src1[off]);
6060  const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 = (const bm::bit_block_t::bunion_t*)(&src2[off]);
6061  const bm::bit_block_t::bunion_t* BMRESTRICT src_u3 = (const bm::bit_block_t::bunion_t*)(&src3[off]);
6063 
6064  bm::id64_t acc = 0;
6065  unsigned j = 0;
6066  do
6067  {
6068  acc |= dst_u->w64[j + 0] &= src_u0->w64[j + 0] & src_u1->w64[j + 0] & src_u2->w64[j + 0] & src_u3->w64[j + 0];
6069  acc |= dst_u->w64[j + 1] &= src_u0->w64[j + 1] & src_u1->w64[j + 1] & src_u2->w64[j + 1] & src_u3->w64[j + 1];
6070  acc |= dst_u->w64[j + 2] &= src_u0->w64[j + 2] & src_u1->w64[j + 2] & src_u2->w64[j + 2] & src_u3->w64[j + 2];
6071  acc |= dst_u->w64[j + 3] &= src_u0->w64[j + 3] & src_u1->w64[j + 3] & src_u2->w64[j + 3] & src_u3->w64[j + 3];
6072  j += 4;
6073  } while (j < bm::set_block_digest_wave_size / 2);
6074 
6075  if (!acc) // all zero
6076  digest &= ~(mask << wave);
6077 #endif
6078 
6079  d = bm::bmi_bslr_u64(d); // d &= d - 1;
6080  } // while
6081 
6082  return digest;
6083 }
6084 
6085 
6086 /*!
6087  \brief digest based bit-block AND
6088 
6089  dst = src1 AND src2
6090 
6091  \param dst - destination block.
6092  \param src1 - source block.
6093  \param src2 - source block.
6094  \param digest - known initial digest
6095 
6096  \return new digest
6097 
6098  @ingroup bitfunc
6099 */
6100 inline
6102  const bm::word_t* BMRESTRICT src1,
6103  const bm::word_t* BMRESTRICT src2,
6104  bm::id64_t digest) BMNOEXCEPT
6105 {
6106  BM_ASSERT(dst);
6107  BM_ASSERT(src1 && src2);
6108  BM_ASSERT(dst != src1 && dst != src2);
6109 
6110  const bm::id64_t mask(1ull);
6111  bm::id64_t d = digest;
6112  while (d)
6113  {
6114  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
6115 
6116  unsigned wave = bm::word_bitcount64(t - 1);
6117  unsigned off = wave * bm::set_block_digest_wave_size;
6118 
6119  #if defined(VECT_AND_DIGEST_2WAY)
6120  bool all_zero = VECT_AND_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
6121  if (all_zero)
6122  digest &= ~(mask << wave);
6123  #else
6124  const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 =
6125  (const bm::bit_block_t::bunion_t*)(&src1[off]);
6126  const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 =
6127  (const bm::bit_block_t::bunion_t*)(&src2[off]);
6129  (bm::bit_block_t::bunion_t*)(&dst[off]);
6130 
6131  bm::id64_t acc = 0;
6132  unsigned j = 0;
6133  do
6134  {
6135  acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & src_u2->w64[j+0];
6136  acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & src_u2->w64[j+1];
6137  acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & src_u2->w64[j+2];
6138  acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & src_u2->w64[j+3];
6139  j+=4;
6140  } while (j < bm::set_block_digest_wave_size/2);
6141 
6142  if (!acc) // all zero
6143  digest &= ~(mask << wave);
6144  #endif
6145 
6146  d = bm::bmi_bslr_u64(d); // d &= d - 1;
6147  } // while
6148 
6149  return digest;
6150 }
6151 
6152 
6153 
6154 /*!
6155  \brief Function ANDs two bitblocks and computes the bitcount.
6156  Function does not analyse availability of source blocks.
6157 
6158  \param src1 - first bit block
6159  \param src2 - second bit block
6160 
6161  @ingroup bitfunc
6162 */
6163 inline
6165  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6166 {
6167  unsigned count;
6168  const bm::word_t* src1_end = src1 + bm::set_block_size;
6169 #ifdef BMVECTOPT
6170  count = VECT_BITCOUNT_AND(src1, src1_end, src2);
6171 #else
6172  count = 0;
6173 # ifdef BM64OPT
6174  const bm::id64_t* b1 = (bm::id64_t*) src1;
6175  const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6176  const bm::id64_t* b2 = (bm::id64_t*) src2;
6177  do
6178  {
6179  count += bitcount64_4way(b1[0] & b2[0],
6180  b1[1] & b2[1],
6181  b1[2] & b2[2],
6182  b1[3] & b2[3]);
6183  b1 += 4;
6184  b2 += 4;
6185  } while (b1 < b1_end);
6186 # else
6187  do
6188  {
6189  BM_INCWORD_BITCOUNT(count, src1[0] & src2[0]);
6190  BM_INCWORD_BITCOUNT(count, src1[1] & src2[1]);
6191  BM_INCWORD_BITCOUNT(count, src1[2] & src2[2]);
6192  BM_INCWORD_BITCOUNT(count, src1[3] & src2[3]);
6193 
6194  src1+=4;
6195  src2+=4;
6196  } while (src1 < src1_end);
6197 # endif
6198 #endif
6199  return count;
6200 }
6201 
6202 
6203 /*!
6204  \brief Function ANDs two bitblocks and tests for any bit.
6205  Function does not analyse availability of source blocks.
6206 
6207  \param src1 - first bit block
6208  \param src2 - second bit block
6209 
6210  @ingroup bitfunc
6211 */
6212 inline
6213 unsigned bit_block_and_any(const bm::word_t* src1,
6214  const bm::word_t* src2) BMNOEXCEPT
6215 {
6216  unsigned count = 0;
6217  const bm::word_t* src1_end = src1 + bm::set_block_size;
6218  do
6219  {
6220  count = (src1[0] & src2[0]) |
6221  (src1[1] & src2[1]) |
6222  (src1[2] & src2[2]) |
6223  (src1[3] & src2[3]);
6224 
6225  src1+=4; src2+=4;
6226  } while ((src1 < src1_end) && !count);
6227  return count;
6228 }
6229 
6230 
6231 
6232 
6233 /*!
6234  \brief Function XORs two bitblocks and computes the bitcount.
6235  Function does not analyse availability of source blocks.
6236 
6237  \param src1 - first bit block
6238  \param src2 - second bit block
6239 
6240  @ingroup bitfunc
6241 */
6242 inline
6244  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6245 {
6246  unsigned count;
6247  const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6248 #ifdef BMVECTOPT
6249  count = VECT_BITCOUNT_XOR(src1, src1_end, src2);
6250 #else
6251  count = 0;
6252 # ifdef BM64OPT
6253  const bm::id64_t* b1 = (bm::id64_t*) src1;
6254  const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6255  const bm::id64_t* b2 = (bm::id64_t*) src2;
6256  do
6257  {
6258  count += bitcount64_4way(b1[0] ^ b2[0],
6259  b1[1] ^ b2[1],
6260  b1[2] ^ b2[2],
6261  b1[3] ^ b2[3]);
6262  b1 += 4;
6263  b2 += 4;
6264  } while (b1 < b1_end);
6265 # else
6266  do
6267  {
6268  BM_INCWORD_BITCOUNT(count, src1[0] ^ src2[0]);
6269  BM_INCWORD_BITCOUNT(count, src1[1] ^ src2[1]);
6270  BM_INCWORD_BITCOUNT(count, src1[2] ^ src2[2]);
6271  BM_INCWORD_BITCOUNT(count, src1[3] ^ src2[3]);
6272 
6273  src1+=4;
6274  src2+=4;
6275  } while (src1 < src1_end);
6276 # endif
6277 #endif
6278  return count;
6279 }
6280 
6281 
6282 /*!
6283  \brief Function XORs two bitblocks and and tests for any bit.
6284  Function does not analyse availability of source blocks.
6285 
6286  \param src1 - first bit block.
6287  \param src2 - second bit block.
6288 
6289  @ingroup bitfunc
6290 */
6291 inline
6293  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6294 {
6295  unsigned count = 0;
6296  const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6297  do
6298  {
6299  count = (src1[0] ^ src2[0]) |
6300  (src1[1] ^ src2[1]) |
6301  (src1[2] ^ src2[2]) |
6302  (src1[3] ^ src2[3]);
6303 
6304  src1+=4; src2+=4;
6305  } while (!count && (src1 < src1_end));
6306  return count;
6307 }
6308 
6309 /*!
6310  \brief Function SUBs two bitblocks and computes the bitcount.
6311  Function does not analyse availability of source blocks.
6312 
6313  \param src1 - first bit block.
6314  \param src2 - second bit block.
6315 
6316  @ingroup bitfunc
6317 */
6318 inline
6320  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6321 {
6322  unsigned count;
6323  const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6324 #ifdef BMVECTOPT
6325  count = VECT_BITCOUNT_SUB(src1, src1_end, src2);
6326 #else
6327  count = 0;
6328 # ifdef BM64OPT
6329  const bm::id64_t* b1 = (bm::id64_t*) src1;
6330  const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6331  const bm::id64_t* b2 = (bm::id64_t*) src2;
6332  do
6333  {
6334  count += bitcount64_4way(b1[0] & ~b2[0],
6335  b1[1] & ~b2[1],
6336  b1[2] & ~b2[2],
6337  b1[3] & ~b2[3]);
6338  b1 += 4;
6339  b2 += 4;
6340  } while (b1 < b1_end);
6341 # else
6342  do
6343  {
6344  BM_INCWORD_BITCOUNT(count, src1[0] & ~src2[0]);
6345  BM_INCWORD_BITCOUNT(count, src1[1] & ~src2[1]);
6346  BM_INCWORD_BITCOUNT(count, src1[2] & ~src2[2]);
6347  BM_INCWORD_BITCOUNT(count, src1[3] & ~src2[3]);
6348 
6349  src1+=4;
6350  src2+=4;
6351  } while (src1 < src1_end);
6352 # endif
6353 #endif
6354  return count;
6355 }
6356 
6357 /*!
6358  \brief Function SUBs two bitblocks and and tests for any bit.
6359  Function does not analyse availability of source blocks.
6360 
6361  \param src1 - first bit block.
6362  \param src2 - second bit block.
6363 
6364  @ingroup bitfunc
6365 */
6366 inline
6368  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6369 {
6370  unsigned count = 0;
6371  const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6372 
6373  do
6374  {
6375  count = (src1[0] & ~src2[0]) |
6376  (src1[1] & ~src2[1]) |
6377  (src1[2] & ~src2[2]) |
6378  (src1[3] & ~src2[3]);
6379 
6380  src1+=4; src2+=4;
6381  } while ((src1 < src1_end) && (count == 0));
6382  return count;
6383 }
6384 
6385 
6386 /*!
6387  \brief Function ORs two bitblocks and computes the bitcount.
6388  Function does not analyse availability of source blocks.
6389 
6390  \param src1 - first bit block
6391  \param src2 - second bit block.
6392 
6393  @ingroup bitfunc
6394 */
6395 inline
6396 unsigned bit_block_or_count(const bm::word_t* src1,
6397  const bm::word_t* src2) BMNOEXCEPT
6398 {
6399  unsigned count;
6400  const bm::word_t* src1_end = src1 + bm::set_block_size;
6401 #ifdef BMVECTOPT
6402  count = VECT_BITCOUNT_OR(src1, src1_end, src2);
6403 #else
6404  count = 0;
6405 # ifdef BM64OPT
6406  const bm::id64_t* b1 = (bm::id64_t*) src1;
6407  const bm::id64_t* b1_end = (bm::id64_t*) src1_end;
6408  const bm::id64_t* b2 = (bm::id64_t*) src2;
6409  do
6410  {
6411  count += bitcount64_4way(b1[0] | b2[0],
6412  b1[1] | b2[1],
6413  b1[2] | b2[2],
6414  b1[3] | b2[3]);
6415  b1 += 4;
6416  b2 += 4;
6417  } while (b1 < b1_end);
6418 # else
6419  do
6420  {
6421  BM_INCWORD_BITCOUNT(count, src1[0] | src2[0]);
6422  BM_INCWORD_BITCOUNT(count, src1[1] | src2[1]);
6423  BM_INCWORD_BITCOUNT(count, src1[2] | src2[2]);
6424  BM_INCWORD_BITCOUNT(count, src1[3] | src2[3]);
6425 
6426  src1+=4;
6427  src2+=4;
6428  } while (src1 < src1_end);
6429 # endif
6430 #endif
6431  return count;
6432 }
6433 
6434 /*!
6435  \brief Function ORs two bitblocks and and tests for any bit.
6436  Function does not analyse availability of source blocks.
6437 
6438  \param src1 - first bit block.
6439  \param src2 - second bit block.
6440 
6441  @ingroup bitfunc
6442 */
6443 inline
6445  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6446 {
6447  unsigned count = 0;
6448  const bm::word_t* BMRESTRICT src1_end = src1 + bm::set_block_size;
6449  do
6450  {
6451  count = (src1[0] | src2[0]) |
6452  (src1[1] | src2[1]) |
6453  (src1[2] | src2[2]) |
6454  (src1[3] | src2[3]);
6455 
6456  src1+=4; src2+=4;
6457  } while (!count && (src1 < src1_end));
6458  return count;
6459 }
6460 
6461 
6462 
6463 
6464 /*!
6465  \brief bitblock AND operation.
6466 
6467  \param dst - destination block.
6468  \param src - source block.
6469 
6470  \returns pointer on destination block.
6471  If returned value equal to src means that block mutation requested.
6472  NULL is valid return value.
6473 
6474  @ingroup bitfunc
6475 */
6477  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6478 {
6479  BM_ASSERT(dst || src);
6480 
6481  bm::word_t* ret = dst;
6482 
6483  if (IS_VALID_ADDR(dst)) // The destination block already exists
6484  {
6485  if (!IS_VALID_ADDR(src))
6486  {
6487  if (IS_EMPTY_BLOCK(src))
6488  {
6489  //If the source block is zero
6490  //just clean the destination block
6491  return 0;
6492  }
6493  }
6494  else
6495  {
6496  // Regular operation AND on the whole block.
6497  //
6498  auto any = bm::bit_block_and(dst, src);
6499  if (!any)
6500  ret = 0;
6501  }
6502  }
6503  else // The destination block does not exist yet
6504  {
6505  if(!IS_VALID_ADDR(src))
6506  {
6507  if(IS_EMPTY_BLOCK(src))
6508  {
6509  // The source block is empty.
6510  // One argument empty - all result is empty.
6511  return 0;
6512  }
6513  // Here we have nothing to do.
6514  // Src block is all ON, dst block remains as it is
6515  }
6516  else // destination block does not exists, src - valid block
6517  {
6518  if (IS_FULL_BLOCK(dst))
6519  return const_cast<bm::word_t*>(src);
6520  // Nothng to do.
6521  // Dst block is all ZERO no combination required.
6522  }
6523  }
6524 
6525  return ret;
6526 }
6527 
6528 
6529 /*!
6530  \brief Performs bitblock AND operation and calculates bitcount of the result.
6531 
6532  \param src1 - first bit block.
6533  \param src2 - second bit block.
6534 
6535  \returns bitcount value
6536 
6537  @ingroup bitfunc
6538 */
6539 inline
6541  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6542 {
6543  if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6544  return 0;
6545  if (src1 == FULL_BLOCK_FAKE_ADDR)
6546  src1 = FULL_BLOCK_REAL_ADDR;
6547  if (src2 == FULL_BLOCK_FAKE_ADDR)
6548  src2 = FULL_BLOCK_REAL_ADDR;
6549 
6550  return bit_block_and_count(src1, src2);
6551 }
6552 
6553 /*!
6554  \brief Performs bitblock AND operation test.
6555 
6556  \param src1 - first bit block.
6557  \param src2 - second bit block.
6558 
6559  \returns non zero if there is any value
6560 
6561  @ingroup bitfunc
6562 */
6563 inline
6565  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6566 {
6567  if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
6568  return 0;
6569  if (src1 == FULL_BLOCK_FAKE_ADDR)
6570  src1 = FULL_BLOCK_REAL_ADDR;
6571  if (src2 == FULL_BLOCK_FAKE_ADDR)
6572  src2 = FULL_BLOCK_REAL_ADDR;
6573  return bit_block_and_any(src1, src2);
6574 }
6575 
6576 
6577 
6578 /*!
6579  \brief Performs bitblock SUB operation and calculates bitcount of the result.
6580 
6581  \param src1 - first bit block.
6582  \param src2 - second bit block
6583 
6584  \returns bitcount value
6585 
6586  @ingroup bitfunc
6587 */
6588 inline
6590  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6591 {
6592  if (src1 == src2)
6593  return 0;
6594  if (IS_EMPTY_BLOCK(src1))
6595  return 0;
6596  if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6597  {
6598  if (IS_FULL_BLOCK(src1))
6599  return bm::gap_max_bits;
6600  return bm::bit_block_count(src1);
6601  }
6602  if (IS_FULL_BLOCK(src2))
6603  return 0;
6604 
6605  if (src1 == FULL_BLOCK_FAKE_ADDR)
6606  src1 = FULL_BLOCK_REAL_ADDR;
6607  if (src2 == FULL_BLOCK_FAKE_ADDR)
6608  src2 = FULL_BLOCK_REAL_ADDR;
6609 
6610  return bm::bit_block_sub_count(src1, src2);
6611 }
6612 
6613 
6614 /*!
6615  \brief Performs inverted bitblock SUB operation and calculates
6616  bitcount of the result.
6617 
6618  \param src1 - first bit block.
6619  \param src2 - second bit block
6620 
6621  \returns bitcount value
6622 
6623  @ingroup bitfunc
6624 */
6625 inline
6627  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6628 {
6629  return bit_operation_sub_count(src2, src1);
6630 }
6631 
6632 
6633 /*!
6634  \brief Performs bitblock test of SUB operation.
6635 
6636  \param src1 - first bit block.
6637  \param src2 - second bit block
6638 
6639  \returns non zero value if there are any bits
6640 
6641  @ingroup bitfunc
6642 */
6643 inline
6645  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6646 {
6647  if (IS_EMPTY_BLOCK(src1))
6648  return 0;
6649  if (src1 == src2)
6650  return 0;
6651 
6652  if (IS_EMPTY_BLOCK(src2)) // nothing to diff
6653  return !bit_is_all_zero(src1);
6654 
6655  if (IS_FULL_BLOCK(src2))
6656  return 0;
6657 
6658  if (src1 == FULL_BLOCK_FAKE_ADDR)
6659  src1 = FULL_BLOCK_REAL_ADDR;
6660  if (src2 == FULL_BLOCK_FAKE_ADDR)
6661  src2 = FULL_BLOCK_REAL_ADDR;
6662 
6663  return bm::bit_block_sub_any(src1, src2);
6664 }
6665 
6666 
6667 
6668 /*!
6669  \brief Performs bitblock OR operation and calculates bitcount of the result.
6670 
6671  \param src1 - first bit block.
6672  \param src2 - second bit block.
6673 
6674  \returns bitcount value
6675 
6676  @ingroup bitfunc
6677 */
6678 inline
6680  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6681 {
6682  if (IS_FULL_BLOCK(src1) || IS_FULL_BLOCK(src2))
6683  return bm::gap_max_bits;
6684 
6685  if (IS_EMPTY_BLOCK(src1))
6686  {
6687  if (!IS_EMPTY_BLOCK(src2))
6688  return bm::bit_block_count(src2);
6689  else
6690  return 0; // both blocks are empty
6691  }
6692  else
6693  {
6694  if (IS_EMPTY_BLOCK(src2))
6695  return bm::bit_block_count(src1);
6696  }
6697  if (src1 == FULL_BLOCK_FAKE_ADDR)
6698  src1 = FULL_BLOCK_REAL_ADDR;
6699  if (src2 == FULL_BLOCK_FAKE_ADDR)
6700  src2 = FULL_BLOCK_REAL_ADDR;
6701 
6702  return bm::bit_block_or_count(src1, src2);
6703 }
6704 
6705 /*!
6706  \brief Performs bitblock OR operation test.
6707 
6708  \param src1 - first bit block.
6709  \param src2 - second bit block.
6710 
6711  \returns non zero value if there are any bits
6712 
6713  @ingroup bitfunc
6714 */
6715 inline
6717  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6718 {
6719  if (IS_EMPTY_BLOCK(src1))
6720  {
6721  if (!IS_EMPTY_BLOCK(src2))
6722  return !bit_is_all_zero(src2);
6723  else
6724  return 0; // both blocks are empty
6725  }
6726  else
6727  {
6728  if (IS_EMPTY_BLOCK(src2))
6729  return !bit_is_all_zero(src1);
6730  }
6731  if (src1 == FULL_BLOCK_FAKE_ADDR)
6732  src1 = FULL_BLOCK_REAL_ADDR;
6733  if (src2 == FULL_BLOCK_FAKE_ADDR)
6734  src2 = FULL_BLOCK_REAL_ADDR;
6735 
6736  return bit_block_or_any(src1, src2);
6737 }
6738 
6739 /*!
6740  \brief Plain bitblock OR operation.
6741  Function does not analyse availability of source and destination blocks.
6742 
6743  \param dst - destination block.
6744  \param src - source block.
6745 
6746  @ingroup bitfunc
6747 */
6748 inline
6750  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6751 {
6752 #ifdef BMVECTOPT
6753  return VECT_OR_BLOCK(dst, src);
6754 #else
6755  const bm::wordop_t* BMRESTRICT wrd_ptr = (wordop_t*)src;
6756  const bm::wordop_t* BMRESTRICT wrd_end = (wordop_t*)(src + bm::set_block_size);
6757  bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6758 
6759  bm::wordop_t acc = 0;
6760  const bm::wordop_t not_acc = acc = ~acc;
6761 
6762  do
6763  {
6764  acc &= (dst_ptr[0] |= wrd_ptr[0]);
6765  acc &= (dst_ptr[1] |= wrd_ptr[1]);
6766  acc &= (dst_ptr[2] |= wrd_ptr[2]);
6767  acc &= (dst_ptr[3] |= wrd_ptr[3]);
6768 
6769  dst_ptr+=4;wrd_ptr+=4;
6770 
6771  } while (wrd_ptr < wrd_end);
6772  return acc == not_acc;
6773 #endif
6774 }
6775 
6776 /*!
6777  \brief 2 way (target := source1 | source2) bitblock OR operation.
6778  \param dst - dest block [out]
6779  \param src1 - source 1
6780  \param src2 - source 2
6781 
6782  @return 1 if produced block of ALL ones
6783  @ingroup bitfunc
6784 */
6785 inline
6787  const bm::word_t* BMRESTRICT src1,
6788  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6789 {
6790 #ifdef BMVECTOPT
6791  return VECT_OR_BLOCK_2WAY(dst, src1, src2);
6792 #else
6793  const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6794  const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6795  const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6796  bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6797 
6798  bm::wordop_t acc = 0;
6799  const bm::wordop_t not_acc = acc = ~acc;
6800  do
6801  {
6802  acc &= (dst_ptr[0] = wrd_ptr1[0] | wrd_ptr2[0]);
6803  acc &= (dst_ptr[1] = wrd_ptr1[1] | wrd_ptr2[1]);
6804  acc &= (dst_ptr[2] = wrd_ptr1[2] | wrd_ptr2[2]);
6805  acc &= (dst_ptr[3] = wrd_ptr1[3] | wrd_ptr2[3]);
6806 
6807  dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6808 
6809  } while (wrd_ptr1 < wrd_end1);
6810  return acc == not_acc;
6811 #endif
6812 }
6813 
6814 
6815 /*!
6816  \brief 2 way (target := source1 ^ source2) bitblock XOR operation.
6817  \param dst - dest block [out]
6818  \param src1 - source 1
6819  \param src2 - source 2
6820 
6821  @return OR accumulator
6822  @ingroup bitfunc
6823 */
6824 inline
6826  const bm::word_t* BMRESTRICT src1,
6827  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6828 {
6829 #ifdef BMVECTOPT
6830  return VECT_XOR_BLOCK_2WAY(dst, src1, src2);
6831 #else
6832  const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6833  const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6834  const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6835  bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6836 
6837  bm::wordop_t acc = 0;
6838  do
6839  {
6840  acc |= (dst_ptr[0] = wrd_ptr1[0] ^ wrd_ptr2[0]);
6841  acc |= (dst_ptr[1] = wrd_ptr1[1] ^ wrd_ptr2[1]);
6842  acc |= (dst_ptr[2] = wrd_ptr1[2] ^ wrd_ptr2[2]);
6843  acc |= (dst_ptr[3] = wrd_ptr1[3] ^ wrd_ptr2[3]);
6844 
6845  dst_ptr+=4; wrd_ptr1+=4; wrd_ptr2+=4;
6846 
6847  } while (wrd_ptr1 < wrd_end1);
6848  return acc;
6849 #endif
6850 }
6851 
6852 
6853 /*!
6854  \brief 3 way (target | source1 | source2) bitblock OR operation.
6855  Function does not analyse availability of source and destination blocks.
6856 
6857  \param dst - sst-dest block [in,out]
6858  \param src1 - source 1
6859  \param src2 - source 2
6860 
6861  @return 1 if produced block of ALL ones
6862 
6863  @ingroup bitfunc
6864 */
6865 inline
6867  const bm::word_t* BMRESTRICT src1,
6868  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
6869 {
6870 #ifdef BMVECTOPT
6871  return VECT_OR_BLOCK_3WAY(dst, src1, src2);
6872 #else
6873  const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6874  const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6875  const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6876  bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6877 
6878  bm::wordop_t acc = 0;
6879  const bm::wordop_t not_acc = acc = ~acc;
6880  do
6881  {
6882  acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0]);
6883  acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1]);
6884  acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2]);
6885  acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3]);
6886 
6887  dst_ptr+=4; wrd_ptr1+=4;wrd_ptr2+=4;
6888 
6889  } while (wrd_ptr1 < wrd_end1);
6890  return acc == not_acc;
6891 #endif
6892 }
6893 
6894 
6895 /*!
6896  \brief 5 way (target, source1, source2) bitblock OR operation.
6897  Function does not analyse availability of source and destination blocks.
6898 
6899  \param dst - destination block.
6900  \param src1 - source1, etc
6901  \param src2 - source1, etc
6902  \param src3 - source1, etc
6903  \param src4 - source1, etc
6904 
6905  @return 1 if produced block of ALL ones
6906 
6907  @ingroup bitfunc
6908 */
6909 inline
6911  const bm::word_t* BMRESTRICT src1,
6912  const bm::word_t* BMRESTRICT src2,
6913  const bm::word_t* BMRESTRICT src3,
6914  const bm::word_t* BMRESTRICT src4) BMNOEXCEPT
6915 {
6916 #ifdef BMVECTOPT
6917  return VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4);
6918 #else
6919  const bm::wordop_t* BMRESTRICT wrd_ptr1 = (wordop_t*)src1;
6920  const bm::wordop_t* BMRESTRICT wrd_end1 = (wordop_t*)(src1 + set_block_size);
6921  const bm::wordop_t* BMRESTRICT wrd_ptr2 = (wordop_t*)src2;
6922  const bm::wordop_t* BMRESTRICT wrd_ptr3 = (wordop_t*)src3;
6923  const bm::wordop_t* BMRESTRICT wrd_ptr4 = (wordop_t*)src4;
6924  bm::wordop_t* BMRESTRICT dst_ptr = (wordop_t*)dst;
6925 
6926  bm::wordop_t acc = 0;
6927  const bm::wordop_t not_acc = acc = ~acc;
6928  do
6929  {
6930  acc &= (dst_ptr[0] |= wrd_ptr1[0] | wrd_ptr2[0] | wrd_ptr3[0] | wrd_ptr4[0]);
6931  acc &= (dst_ptr[1] |= wrd_ptr1[1] | wrd_ptr2[1] | wrd_ptr3[1] | wrd_ptr4[1]);
6932  acc &= (dst_ptr[2] |= wrd_ptr1[2] | wrd_ptr2[2] | wrd_ptr3[2] | wrd_ptr4[2]);
6933  acc &= (dst_ptr[3] |= wrd_ptr1[3] | wrd_ptr2[3] | wrd_ptr3[3] | wrd_ptr4[3]);
6934 
6935  dst_ptr+=4;
6936  wrd_ptr1+=4;wrd_ptr2+=4;wrd_ptr3+=4;wrd_ptr4+=4;
6937 
6938  } while (wrd_ptr1 < wrd_end1);
6939  return acc == not_acc;
6940 #endif
6941 }
6942 
6943 
6944 
6945 
6946 /*!
6947  \brief Block OR operation. Makes analysis if block is 0 or FULL.
6948 
6949  \param dst - destination block.
6950  \param src - source block.
6951 
6952  \returns pointer on destination block.
6953  If returned value equal to src means that block mutation requested.
6954  NULL is valid return value.
6955 
6956  @ingroup bitfunc
6957 */
6958 inline
6960  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
6961 {
6962  BM_ASSERT(dst || src);
6963 
6964  bm::word_t* ret = dst;
6965 
6966  if (IS_VALID_ADDR(dst)) // The destination block already exists
6967  {
6968  if (!IS_VALID_ADDR(src))
6969  {
6970  if (IS_FULL_BLOCK(src))
6971  {
6972  // if the source block is all set
6973  // just set the destination block
6974  ::memset(dst, 0xFF, bm::set_block_size * sizeof(bm::word_t));
6975  }
6976  }
6977  else
6978  {
6979  // Regular operation OR on the whole block
6980  bm::bit_block_or(dst, src);
6981  }
6982  }
6983  else // The destination block does not exist yet
6984  {
6985  if (!IS_VALID_ADDR(src))
6986  {
6987  if (IS_FULL_BLOCK(src))
6988  {
6989  // The source block is all set, because dst does not exist
6990  // we can simply replace it.
6991  return const_cast<bm::word_t*>(FULL_BLOCK_FAKE_ADDR);
6992  }
6993  }
6994  else
6995  {
6996  if (dst == 0)
6997  {
6998  // The only case when we have to allocate the new block:
6999  // Src is all zero and Dst does not exist
7000  return const_cast<bm::word_t*>(src);
7001  }
7002  }
7003  }
7004  return ret;
7005 }
7006 
7007 /*!
7008  \brief Plain bitblock SUB (AND NOT) operation.
7009  Function does not analyse availability of source and destination blocks.
7010 
7011  \param dst - destination block.
7012  \param src - source block.
7013 
7014  \return 0 if SUB operation did not produce anything (no 1s in the output)
7015 
7016  @ingroup bitfunc
7017 */
7018 inline
7020  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7021 {
7022 #ifdef BMVECTOPT
7023  bm::id64_t acc = VECT_SUB_BLOCK(dst, src);
7024  return acc;
7025 #else
7026  unsigned arr_sz = bm::set_block_size / 2;
7027 
7030 
7031  bm::id64_t acc = 0;
7032  for (unsigned i = 0; i < arr_sz; i+=4)
7033  {
7034  acc |= dst_u->w64[i] &= ~src_u->w64[i];
7035  acc |= dst_u->w64[i+1] &= ~src_u->w64[i+1];
7036  acc |= dst_u->w64[i+2] &= ~src_u->w64[i+2];
7037  acc |= dst_u->w64[i+3] &= ~src_u->w64[i+3];
7038  }
7039  return acc;
7040 #endif
7041 }
7042 
7043 /*!
7044  \brief digest based bitblock SUB (AND NOT) operation
7045 
7046  \param dst - destination block.
7047  \param src - source block.
7048  \param digest - known digest of dst block
7049 
7050  \return new digest
7051 
7052  @ingroup bitfunc
7053 */
7054 inline
7056  const bm::word_t* BMRESTRICT src,
7057  bm::id64_t digest) BMNOEXCEPT
7058 {
7059  BM_ASSERT(dst);
7060  BM_ASSERT(src);
7061  BM_ASSERT(dst != src);
7062 
7063  const bm::id64_t mask(1ull);
7064 
7065  bm::id64_t d = digest;
7066  while (d)
7067  {
7068  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7069 
7070  unsigned wave = bm::word_bitcount64(t - 1);
7071  unsigned off = wave * bm::set_block_digest_wave_size;
7072 
7073  #if defined(VECT_SUB_DIGEST)
7074  bool all_zero = VECT_SUB_DIGEST(&dst[off], &src[off]);
7075  if (all_zero)
7076  digest &= ~(mask << wave);
7077  #else
7078  const bm::bit_block_t::bunion_t* BMRESTRICT src_u = (const bm::bit_block_t::bunion_t*)(&src[off]);
7080 
7081  bm::id64_t acc = 0;
7082  unsigned j = 0;
7083  do
7084  {
7085  acc |= dst_u->w64[j+0] &= ~src_u->w64[j+0];
7086  acc |= dst_u->w64[j+1] &= ~src_u->w64[j+1];
7087  acc |= dst_u->w64[j+2] &= ~src_u->w64[j+2];
7088  acc |= dst_u->w64[j+3] &= ~src_u->w64[j+3];
7089  j+=4;
7090  } while (j < bm::set_block_digest_wave_size/2);
7091 
7092  if (!acc) // all zero
7093  digest &= ~(mask << wave);
7094  #endif
7095 
7096  d = bm::bmi_bslr_u64(d); // d &= d - 1;
7097  } // while
7098 
7099  return digest;
7100 }
7101 
7102 /*!
7103  \brief digest based bitblock SUB (AND NOT) operation (3 operand)
7104 
7105  \param dst - destination block.
7106  \param src1 - source block 1
7107  \param src2 - source block 2
7108  \param digest - known digest of dst block
7109 
7110  \return new digest
7111 
7112  @ingroup bitfunc
7113 */
7114 inline
7116  const bm::word_t* BMRESTRICT src1,
7117  const bm::word_t* BMRESTRICT src2,
7118  bm::id64_t digest) BMNOEXCEPT
7119 {
7120  BM_ASSERT(dst);
7121  BM_ASSERT(src1 && src2);
7122  BM_ASSERT(dst != src1 && dst != src2);
7123 
7124  const bm::id64_t mask(1ull);
7125 
7126  bm::id64_t d = digest;
7127  while (d)
7128  {
7129  bm::id64_t t = bm::bmi_blsi_u64(d); // d & -d;
7130 
7131  unsigned wave = bm::word_bitcount64(t - 1);
7132  unsigned off = wave * bm::set_block_digest_wave_size;
7133 
7134  #if defined(VECT_SUB_DIGEST_2WAY)
7135  bool all_zero = VECT_SUB_DIGEST_2WAY(&dst[off], &src1[off], &src2[off]);
7136  if (all_zero)
7137  digest &= ~(mask << wave);
7138  #else
7139  const bm::bit_block_t::bunion_t* BMRESTRICT src_u1 =
7140  (const bm::bit_block_t::bunion_t*)(&src1[off]);
7141  const bm::bit_block_t::bunion_t* BMRESTRICT src_u2 =
7142  (const bm::bit_block_t::bunion_t*)(&src2[off]);
7144  (bm::bit_block_t::bunion_t*)(&dst[off]);
7145 
7146  bm::id64_t acc = 0;
7147  unsigned j = 0;
7148  do
7149  {
7150  acc |= dst_u->w64[j+0] = src_u1->w64[j+0] & ~src_u2->w64[j+0];
7151  acc |= dst_u->w64[j+1] = src_u1->w64[j+1] & ~src_u2->w64[j+1];
7152  acc |= dst_u->w64[j+2] = src_u1->w64[j+2] & ~src_u2->w64[j+2];
7153  acc |= dst_u->w64[j+3] = src_u1->w64[j+3] & ~src_u2->w64[j+3];
7154  j+=4;
7155  } while (j < bm::set_block_digest_wave_size/2);
7156 
7157  if (!acc) // all zero
7158  digest &= ~(mask << wave);
7159  #endif
7160 
7161  d = bm::bmi_bslr_u64(d); // d &= d - 1;
7162  } // while
7163 
7164  return digest;
7165 }
7166 
7167 
7168 
7169 /*!
7170  \brief bitblock SUB operation.
7171 
7172  \param dst - destination block.
7173  \param src - source block.
7174 
7175  \returns pointer on destination block.
7176  If returned value equal to src means that block mutation requested.
7177  NULL is valid return value.
7178 
7179  @ingroup bitfunc
7180 */
7181 inline
7183  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7184 {
7185  BM_ASSERT(dst || src);
7186 
7187  bm::word_t* ret = dst;
7188  if (IS_VALID_ADDR(dst)) // The destination block already exists
7189  {
7190  if (!IS_VALID_ADDR(src))
7191  {
7192  if (IS_FULL_BLOCK(src))
7193  {
7194  // If the source block is all set
7195  // just clean the destination block
7196  return 0;
7197  }
7198  }
7199  else
7200  {
7201  auto any = bm::bit_block_sub(dst, src);
7202  if (!any)
7203  ret = 0;
7204  }
7205  }
7206  else // The destination block does not exist yet
7207  {
7208  if (!IS_VALID_ADDR(src))
7209  {
7210  if (IS_FULL_BLOCK(src))
7211  {
7212  // The source block is full
7213  return 0;
7214  }
7215  }
7216  else
7217  {
7218  if (IS_FULL_BLOCK(dst))
7219  {
7220  // The only case when we have to allocate the new block:
7221  // dst is all set and src exists
7222  return const_cast<bm::word_t*>(src);
7223  }
7224  }
7225  }
7226  return ret;
7227 }
7228 
7229 
7230 /*!
7231  \brief Plain bitblock XOR operation.
7232  Function does not analyse availability of source and destination blocks.
7233 
7234  \param dst - destination block.
7235  \param src - source block.
7236 
7237  @ingroup bitfunc
7238 */
7239 inline
7241  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7242 {
7243  BM_ASSERT(dst);
7244  BM_ASSERT(src);
7245  BM_ASSERT(dst != src);
7246 
7247 #ifdef BMVECTOPT
7248  bm::id64_t acc = VECT_XOR_BLOCK(dst, src);
7249 #else
7250  unsigned arr_sz = bm::set_block_size / 2;
7251 
7254 
7255  bm::id64_t acc = 0;
7256  for (unsigned i = 0; i < arr_sz; i+=4)
7257  {
7258  acc |= dst_u->w64[i] ^= src_u->w64[i];
7259  acc |= dst_u->w64[i+1] ^= src_u->w64[i+1];
7260  acc |= dst_u->w64[i+2] ^= src_u->w64[i+2];
7261  acc |= dst_u->w64[i+3] ^= src_u->w64[i+3];
7262  }
7263 #endif
7264  return acc;
7265 }
7266 
7267 /*!
7268  \brief bitblock AND NOT with constant ~0 mask operation.
7269 
7270  \param dst - destination block.
7271  \param src - source block.
7272 
7273  @ingroup bitfunc
7274 */
7275 inline
7277  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7278 {
7279  const bm::word_t* BMRESTRICT src_end = src + bm::set_block_size;
7280 #ifdef BMVECTOPT
7281  VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, ~0u);
7282 #else
7283  bm::wordop_t* dst_ptr = (wordop_t*)dst;
7284  const bm::wordop_t* wrd_ptr = (wordop_t*) src;
7285  const bm::wordop_t* wrd_end = (wordop_t*) src_end;
7286 
7287  do
7288  {
7289  dst_ptr[0] = bm::all_bits_mask & ~wrd_ptr[0];
7290  dst_ptr[1] = bm::all_bits_mask & ~wrd_ptr[1];
7291  dst_ptr[2] = bm::all_bits_mask & ~wrd_ptr[2];
7292  dst_ptr[3] = bm::all_bits_mask & ~wrd_ptr[3];
7293  dst_ptr+=4; wrd_ptr+=4;
7294  } while (wrd_ptr < wrd_end);
7295 #endif
7296 }
7297 
7298 /*!
7299  \brief bitblock XOR operation.
7300 
7301  \param dst - destination block.
7302  \param src - source block.
7303 
7304  \returns pointer on destination block.
7305  If returned value equal to src means that block mutation requested.
7306  NULL is valid return value.
7307 
7308  @ingroup bitfunc
7309 */
7310 inline
7312  const bm::word_t* BMRESTRICT src) BMNOEXCEPT
7313 {
7314  BM_ASSERT(dst || src);
7315  if (src == dst) return 0; // XOR rule
7316 
7317  bm::word_t* ret = dst;
7318 
7319  if (IS_VALID_ADDR(dst)) // The destination block already exists
7320  {
7321  if (!src) return dst;
7322 
7323  bit_block_xor(dst, src);
7324  }
7325  else // The destination block does not exist yet
7326  {
7327  if (!src) return dst; // 1 xor 0 = 1
7328 
7329  // Here we have two cases:
7330  // if dest block is full or zero if zero we need to copy the source
7331  // otherwise XOR loop against 0xFF...
7332  //BM_ASSERT(dst == 0);
7333  return const_cast<bm::word_t*>(src); // src is the final result
7334  }
7335  return ret;
7336 }
7337 
7338 /*!
7339  \brief Performs bitblock XOR operation and calculates bitcount of the result.
7340 
7341  \param src1 - bit block start ptr
7342  \param src2 - second bit block
7343 
7344  \returns bitcount value
7345 
7346  @ingroup bitfunc
7347 */
7348 inline
7350  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7351 {
7352  if (src1 == src2)
7353  return 0;
7354 
7355  if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7356  {
7357  const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7358  if (IS_FULL_BLOCK(block))
7359  return bm::gap_max_bits;
7360  return bm::bit_block_count(block);
7361  }
7362  if (src1 == FULL_BLOCK_FAKE_ADDR)
7363  src1 = FULL_BLOCK_REAL_ADDR;
7364  if (src2 == FULL_BLOCK_FAKE_ADDR)
7365  src2 = FULL_BLOCK_REAL_ADDR;
7366 
7367  return bit_block_xor_count(src1, src2);
7368 }
7369 
7370 /*!
7371  \brief Performs bitblock XOR operation test.
7372 
7373  \param src1 - bit block start ptr
7374  \param src2 - second bit block ptr
7375 
7376  \returns non zero value if there are bits
7377 
7378  @ingroup bitfunc
7379 */
7380 inline
7382  const bm::word_t* BMRESTRICT src2) BMNOEXCEPT
7383 {
7384  if (src1 == src2)
7385  return 0;
7386  if (IS_EMPTY_BLOCK(src1) || IS_EMPTY_BLOCK(src2))
7387  {
7388  const bm::word_t* block = IS_EMPTY_BLOCK(src1) ? src2 : src1;
7389  return !bit_is_all_zero(block);
7390  }
7391  return bm::bit_block_xor_any(src1, src2);
7392 }
7393 
7394 
7395 
7396 /**
7397  \brief Inspects block for full zero words
7398 
7399  \param blk - bit block pointer
7400  \param data_size - data size
7401 
7402  \return size of all non-zero words
7403 
7404  @ingroup bitfunc
7405 */
7406 template<class T>
7407 unsigned bit_count_nonzero_size(const T* blk, unsigned data_size) BMNOEXCEPT
7408 {
7409  BM_ASSERT(blk && data_size);
7410  unsigned count = 0;
7411  const T* blk_end = blk + data_size - 2;
7412  do
7413  {
7414  if (*blk == 0)
7415  {
7416  // scan fwd to find 0 island length
7417  const T* blk_j = blk + 1;
7418  for (; blk_j < blk_end; ++blk_j)
7419  {
7420  if (*blk_j != 0)
7421  break;
7422  }
7423  blk = blk_j-1;
7424  count += (unsigned)sizeof(gap_word_t);
7425  }
7426  else
7427  {
7428  // scan fwd to find non-0 island length
7429  const T* blk_j = blk + 1;
7430  for ( ; blk_j < blk_end; ++blk_j)
7431  {
7432  if (*blk_j == 0)
7433  {
7434  // look ahead to identify and ignore short 0-run
7435  if (blk_j[1] | blk_j[2])
7436  {
7437  // skip zero word
7438  ++blk_j;
7439  continue;
7440  }
7441  break;
7442  }
7443  }
7444  count += unsigned(sizeof(gap_word_t));
7445  // count all bit-words now
7446  count += unsigned(blk_j - blk) * unsigned(sizeof(T));
7447  blk = blk_j;
7448  }
7449  ++blk;
7450  }
7451  while(blk < blk_end);
7452  return count + unsigned(2 * sizeof(T));
7453 }
7454 
7455 
7456 /**
7457  \brief Searches for the next 1 bit in the BIT block
7458  \param block - BIT buffer
7459  \param nbit - bit index to start checking from
7460  \param pos - [out] found value
7461 
7462  \return 0 if not found
7463 
7464  @ingroup bitfunc
7465 */
7466 inline
7467 unsigned bit_block_find(const bm::word_t* BMRESTRICT block,
7468  unsigned nbit, unsigned* BMRESTRICT pos) BMNOEXCEPT
7469 {
7470  BM_ASSERT(block);
7471  BM_ASSERT(pos);
7472 
7473  unsigned nword = unsigned(nbit >> bm::set_word_shift);
7474  unsigned bit_pos = (nbit & bm::set_word_mask);
7475 
7476  bm::word_t w = block[nword];
7477  w &= (1u << bit_pos);
7478  if (w)
7479  {
7480  *pos = nbit;
7481  return 1;
7482  }
7483  w = block[nword] >> bit_pos;
7484  w <<= bit_pos; // clear the trailing bits
7485  if (w)
7486  {
7487  bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7488  *pos = unsigned(bit_pos + (nword * 8u * unsigned(sizeof(bm::word_t))));
7489  return 1;
7490  }
7491 
7492  for (unsigned i = nword+1; i < bm::set_block_size; ++i)
7493  {
7494  w = block[i];
7495  if (w)
7496  {
7497  bit_pos = bm::bit_scan_forward32(w); // trailing zeros
7498  *pos = unsigned(bit_pos + (i * 8u * unsigned(sizeof(bm::word_t))));
7499  return w;
7500  }
7501  } // for i
7502  return 0u;
7503 }
7504 
7505 
7506 
7507 
7508 /*!
7509  \brief BIT block find the last set bit (backward search)
7510 
7511  \param block - bit block buffer pointer
7512  \param last - index of the last 1 bit (out)
7513  \return true if found
7514 
7515  @ingroup bitfunc
7516 */
7517 inline
7518 unsigned bit_find_last(const bm::word_t* BMRESTRICT block,
7519  unsigned* BMRESTRICT last) BMNOEXCEPT
7520 {
7521  BM_ASSERT(block);
7522  BM_ASSERT(last);
7523 
7524  // TODO: SIMD version
7525 
7526  for (unsigned i = bm::set_block_size-1; true; --i)
7527  {
7528  bm::word_t w = block[i];
7529  if (w)
7530  {
7531  unsigned idx = bm::bit_scan_reverse(w);
7532  *last = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7533  return w;
7534  }
7535  if (i == 0)
7536  break;
7537  } // for i
7538  return 0u;
7539 }
7540 
7541 /*!
7542  \brief BIT block find the first set bit
7543 
7544  \param block - bit block buffer pointer
7545  \param pos - index of the first 1 bit (out)
7546  \return 0 if not found
7547 
7548  @ingroup bitfunc
7549  @internal
7550 */
7551 inline
7553  unsigned* BMRESTRICT pos) BMNOEXCEPT
7554 {
7555  BM_ASSERT(block);
7556  BM_ASSERT(pos);
7557 
7558 #ifdef VECT_BIT_FIND_FIRST
7559  return VECT_BIT_FIND_FIRST(block, pos);
7560 #else
7561  for (unsigned i = 0; i < bm::set_block_size; ++i)
7562  {
7563  bm::word_t w = block[i];
7564  if (w)
7565  {
7566  unsigned idx = bm::bit_scan_forward32(w); // trailing zeros
7567  *pos = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7568  return w;
7569  }
7570  } // for i
7571  return false;
7572 #endif
7573 }
7574 
7575 /*!
7576  \brief BIT block find the first set bit
7577 
7578  \param block - bit block buffer pointer
7579  \param first - index of the first 1 bit (out)
7580  \param digest - known digest of dst block
7581 
7582  \return 0 if not found
7583 
7584  @ingroup bitfunc
7585 */
7586 inline
7587 unsigned bit_find_first(const bm::word_t* BMRESTRICT block,
7588  unsigned* BMRESTRICT first,
7589  bm::id64_t digest) BMNOEXCEPT
7590 {
7591  BM_ASSERT(block);
7592  BM_ASSERT(first);
7593  BM_ASSERT(digest);
7594 
7595  bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7596 
7597  unsigned wave = bm::word_bitcount64(t - 1);
7598  unsigned off = wave * bm::set_block_digest_wave_size;
7599  for (unsigned i = off; i < bm::set_block_size; ++i)
7600  {
7601  bm::word_t w = block[i];
7602  if (w)
7603  {
7604  unsigned idx = bit_scan_forward32(w); // trailing zeros
7605  *first = unsigned(idx + (i * 8u * unsigned(sizeof(bm::word_t))));
7606  return w;
7607  }
7608  } // for i
7609  return 0u;
7610 }
7611 
7612 
7613 /*!
7614  \brief BIT block find the first set bit if only 1 bit is set
7615 
7616  \param block - bit block buffer pointer
7617  \param first - index of the first 1 bit (out)
7618  \param digest - known digest of dst block
7619 
7620  \return 0 if not found
7621 
7622  @ingroup bitfunc
7623 */
7624 inline
7626  unsigned* BMRESTRICT first,
7627  bm::id64_t digest) BMNOEXCEPT
7628 {
7629  BM_ASSERT(block);
7630  BM_ASSERT(first);
7631 
7632  unsigned bc = bm::word_bitcount64(digest);
7633  if (bc != 1)
7634  return false;
7635 
7636  bool found = false;
7637  bm::id64_t t = bm::bmi_blsi_u64(digest); // d & -d;
7638 
7639  unsigned wave = bm::word_bitcount64(t - 1);
7640  unsigned off = wave * bm::set_block_digest_wave_size;
7641  unsigned i;
7642  for (i = off; i < off + bm::set_block_digest_wave_size; ++i)
7643  {
7644  bm::word_t w = block[i];
7645  if (w)
7646  {
7647  bc = bm::word_bitcount(w);
7648  if (bc != 1)
7649  return false;
7650 
7651  unsigned idx = bit_scan_forward32(w); // trailing zeros
7652  *first = unsigned(idx + (i * 8u * sizeof(bm::word_t)));
7653  found = true;
7654  break;
7655  }
7656  } // for i
7657 
7658  // check if all other bits are zero
7659  for (++i; i < off + bm::set_block_digest_wave_size; ++i)
7660  {
7661  if (block[i])
7662  return false;
7663  }
7664  return found;
7665 }
7666 
7667 
7668 /*!
7669  \brief BIT block find position for the rank
7670 
7671  \param block - bit block buffer pointer
7672  \param rank - rank to find (must be > 0)
7673  \param nbit_from - start bit position in block
7674  \param nbit_pos - (out)found position
7675 
7676  \return 0 if position with rank was found, or
7677  the remaining rank (rank - population count)
7678 
7679  @ingroup bitfunc
7680 */
7681 template<typename SIZE_TYPE>
7682 SIZE_TYPE bit_find_rank(const bm::word_t* const block,
7683  SIZE_TYPE rank,
7684  unsigned nbit_from,
7685  unsigned& nbit_pos) BMNOEXCEPT
7686 {
7687  BM_ASSERT(block);
7688  BM_ASSERT(rank);
7689 
7690  unsigned nword = nbit_from >> bm::set_word_shift;
7691  BM_ASSERT(nword < bm::set_block_size);
7692 
7693  unsigned pos = nbit_from;
7694  bm::id_t nbit = (nbit_from & bm::set_word_mask);
7695 
7696  if (nbit)
7697  {
7698  bm::id_t w = block[nword];
7699  w >>= nbit;
7700  unsigned bc = bm::word_bitcount(w);
7701  if (bc < rank) // skip this
7702  {
7703  rank -= bc; pos += unsigned(32u - nbit);
7704  ++nword;
7705  }
7706  else // target word
7707  {
7708  unsigned idx = bm::word_select64(w, unsigned(rank));
7709  nbit_pos = pos + idx;
7710  return 0;
7711  }
7712  }
7713 
7714  if (bm::conditional<sizeof(void*) == 8>::test()) // 64-bit fast scan
7715  {
7716  for (; nword < bm::set_block_size-1; nword+=2)
7717  {
7718  bm::id64_t w = (bm::id64_t(block[nword+1]) << 32) | bm::id64_t(block[nword]);
7719  bm::id_t bc = bm::word_bitcount64(w);
7720  if (bc >= rank) // target
7721  {
7722  unsigned idx = bm::word_select64(w, unsigned(rank));
7723  nbit_pos = pos + idx;
7724  return 0;
7725  }
7726  rank -= bc;
7727  pos += 64u;
7728  }
7729  }
7730 
7731  for (; nword < bm::set_block_size; ++nword)
7732  {
7733  bm::id_t w = block[nword];
7734  bm::id_t bc = bm::word_bitcount(w);
7735  if (rank > bc)
7736  {
7737  rank -= bc; pos += 32u;
7738  continue;
7739  }
7740  unsigned idx = bm::word_select64(w, unsigned(rank));
7741  nbit_pos = pos + idx;
7742  return 0;
7743  } // for nword
7744  return rank;
7745 }
7746 
7747 /**
7748  \brief Find rank in block (GAP or BIT)
7749 
7750  \param block - bit block buffer pointer
7751  \param rank - rank to find (must be > 0)
7752  \param nbit_from - start bit position in block
7753  \param nbit_pos - found position
7754 
7755  \return 0 if position with rank was found, or
7756  the remaining rank (rank - population count)
7757 
7758  @internal
7759 */
7760 template<typename SIZE_TYPE>
7761 SIZE_TYPE block_find_rank(const bm::word_t* const block,
7762  SIZE_TYPE rank,
7763  unsigned nbit_from,
7764  unsigned& nbit_pos) BMNOEXCEPT
7765 {
7766  if (BM_IS_GAP(block))
7767  {
7768  const bm::gap_word_t* const gap_block = BMGAP_PTR(block);
7769  rank = bm::gap_find_rank(gap_block, rank, nbit_from, nbit_pos);
7770  }
7771  else
7772  {
7773  rank = bm::bit_find_rank(block, rank, nbit_from, nbit_pos);
7774  }
7775  return rank;
7776 }
7777 
7778 
7779 
7780 /*!
7781  @brief Choose best representation for a bit-block
7782  @ingroup bitfunc
7783 */
7784 inline
7786  unsigned total_possible_bitcount,
7787  unsigned gap_count,
7788  unsigned block_size) BMNOEXCEPT
7789 {
7790  unsigned arr_size = unsigned(sizeof(bm::gap_word_t) * bit_count + sizeof(bm::gap_word_t));
7791  unsigned gap_size = unsigned(sizeof(bm::gap_word_t) * gap_count + sizeof(bm::gap_word_t));
7792  unsigned inv_arr_size = unsigned(sizeof(bm::gap_word_t) * (total_possible_bitcount - bit_count) + sizeof(bm::gap_word_t));
7793 
7794  if ((gap_size < block_size) && (gap_size < arr_size) && (gap_size < inv_arr_size))
7795  {
7796  return bm::set_gap;
7797  }
7798 
7799  if (arr_size < inv_arr_size)
7800  {
7801  if ((arr_size < block_size) && (arr_size < gap_size))
7802  {
7803  return bm::set_array1;
7804  }
7805  }
7806  else
7807  {
7808  if ((inv_arr_size < block_size) && (inv_arr_size < gap_size))
7809  {
7810  return bm::set_array0;
7811  }
7812  }
7813  return bm::set_bitset;
7814 }
7815 
7816 /*!
7817  @brief Convert bit block into an array of ints corresponding to 1 bits
7818  @return destination size as a result of block decoding
7819  @ingroup bitfunc
7820 */
7821 template<typename T>
7823  const unsigned* BMRESTRICT src,
7824  bm::id_t bits,
7825  unsigned dest_len,
7826  unsigned mask = 0) BMNOEXCEPT
7827 {
7828  T* BMRESTRICT pcurr = dest;
7829  for (unsigned bit_idx=0; bit_idx < bits;
7830  ++src,bit_idx += unsigned(sizeof(*src) * 8))
7831  {
7832  unsigned val = *src ^ mask; // invert value by XOR 0xFF..
7833  if (val == 0)
7834  continue;
7835  if (pcurr + unsigned(sizeof(val)*8) >= dest + dest_len) // insufficient space
7836  return 0;
7837  // popscan loop to decode bits in a word
7838  while (val)
7839  {
7840  bm::id_t t = val & -val;
7841  *pcurr++ = (T)(bm::word_bitcount(t - 1) + bit_idx);
7842  val &= val - 1;
7843  }
7844  } // for
7845  return (T)(pcurr - dest);
7846 }
7847 
7848 /**
7849  \brief Checks all conditions and returns true if block consists of only 0 bits
7850  \param blk - Blocks's pointer
7851  \param deep_scan - flag to do full bit block verification (scan)
7852  when deep scan is not requested result can be approximate
7853  \returns true if all bits are in the block are 0
7854 
7855  @internal
7856 */
7857 inline
7858 bool check_block_zero(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7859 {
7860  if (!blk) return true;
7861  if (IS_FULL_BLOCK(blk)) return false;
7862 
7863  bool ret;
7864  if (BM_IS_GAP(blk))
7865  ret = gap_is_all_zero(BMGAP_PTR(blk));
7866  else
7867  ret = deep_scan ? bm::bit_is_all_zero(blk) : 0;
7868  return ret;
7869 }
7870 
7871 
7872 /**
7873  \brief Checks if block has only 1 bits
7874  \param blk - Block's pointer
7875  \param deep_scan - flag to do full bit block verification (scan)
7876  when deep scan is not requested result can be approximate
7877  \return true if block consists of 1 bits.
7878 
7879  @internal
7880 */
7881 inline
7882 bool check_block_one(const bm::word_t* blk, bool deep_scan) BMNOEXCEPT
7883 {
7884  if (blk == 0) return false;
7885 
7886  if (BM_IS_GAP(blk))
7887  return bm::gap_is_all_one(BMGAP_PTR(blk));
7888 
7889  if (IS_FULL_BLOCK(blk))
7890  return true;
7891 
7892  if (!deep_scan)
7893  return false; // block exists - presume it has 0 bits
7894 
7895  return bm::is_bits_one((wordop_t*)blk);
7896 }
7897 
7898 
7899 
7900 /*! @brief Calculates memory overhead for number of gap blocks sharing
7901  the same memory allocation table (level lengths table).
7902  @ingroup gapfunc
7903 */
7904 template<typename T>
7905 unsigned gap_overhead(const T* length,
7906  const T* length_end,
7907  const T* glevel_len) BMNOEXCEPT
7908 {
7909  BM_ASSERT(length && length_end && glevel_len);
7910 
7911  unsigned overhead = 0;
7912  for (;length < length_end; ++length)
7913  {
7914  unsigned len = *length;
7915  int level = gap_calc_level(len, glevel_len);
7916  BM_ASSERT(level >= 0 && level < (int)bm::gap_levels);
7917  unsigned capacity = glevel_len[level];
7918  BM_ASSERT(capacity >= len);
7919  overhead += capacity - len;
7920  }
7921  return overhead;
7922 }
7923 
7924 
7925 /*! @brief Finds optimal gap blocks lengths.
7926  @param length - first element of GAP lengths array
7927  @param length_end - end of the GAP lengths array
7928  @param glevel_len - destination GAP lengths array
7929  @ingroup gapfunc
7930 */
7931 template<typename T>
7932 bool improve_gap_levels(const T* length,
7933  const T* length_end,
7934  T* glevel_len) BMNOEXCEPT
7935 {
7936  BM_ASSERT(length && length_end && glevel_len);
7937 
7938  size_t lsize = size_t(length_end - length);
7939 
7940  BM_ASSERT(lsize);
7941 
7942  gap_word_t max_len = 0;
7943  unsigned i;
7944  for (i = 0; i < lsize; ++i)
7945  {
7946  if (length[i] > max_len)
7947  max_len = length[i];
7948  }
7949  if (max_len < 5 || lsize <= bm::gap_levels)
7950  {
7951  glevel_len[0] = T(max_len + 4);
7952  for (i = 1; i < bm::gap_levels; ++i)
7953  {
7954  glevel_len[i] = bm::gap_max_buff_len;
7955  }
7956  return true;
7957  }
7958 
7959  glevel_len[bm::gap_levels-1] = T(max_len + 5);
7960 
7961  unsigned min_overhead = gap_overhead(length, length_end, glevel_len);
7962  bool is_improved = false;
7963 
7964  // main problem solving loop
7965  //
7966  for (i = bm::gap_levels-2; ; --i)
7967  {
7968  unsigned opt_len = 0;
7969  unsigned j;
7970  bool imp_flag = false;
7971  gap_word_t gap_saved_value = glevel_len[i];
7972  for (j = 0; j < lsize; ++j)
7973  {
7974  glevel_len[i] = T(length[j] + 4);
7975  unsigned ov = gap_overhead(length, length_end, glevel_len);
7976  if (ov <= min_overhead)
7977  {
7978  min_overhead = ov;
7979  opt_len = length[j]+4;
7980  imp_flag = true;
7981  }
7982  }
7983  if (imp_flag)
7984  {
7985  glevel_len[i] = (T)opt_len; // length[opt_idx]+4;
7986  is_improved = true;
7987  }
7988  else
7989  {
7990  glevel_len[i] = gap_saved_value;
7991  }
7992  if (i == 0)
7993  break;
7994  }
7995 
7996  // Remove duplicates
7997  //
7998  T val = *glevel_len;
7999  T* gp = glevel_len;
8000  T* res = glevel_len;
8001  for (i = 0; i < bm::gap_levels; ++i)
8002  {
8003  if (val != *gp)
8004  {
8005  val = *gp;
8006  *++res = val;
8007  }
8008  ++gp;
8009  }
8010 
8011  // Filling the "unused" part with max. possible value
8012  while (++res < (glevel_len + bm::gap_levels))
8013  {
8014  *res = bm::gap_max_buff_len;
8015  }
8016 
8017  return is_improved;
8018 
8019 }
8020 
8021 /*!
8022  \brief Find first bit which is different between two blocks (GAP or bit)
8023  \param blk - block 1
8024  \param arg_blk - block 2
8025  \param pos - out - position of difference (undefined if blocks are equal)
8026  \return true if difference was found
8027  @internal
8028 */
8029 inline
8031  const bm::word_t* BMRESTRICT arg_blk,
8032  unsigned* BMRESTRICT pos) BMNOEXCEPT
8033 {
8034  // If one block is zero we check if the other one has at least
8035  // one bit ON
8036 
8037  if (!blk || !arg_blk)
8038  {
8039  const bm::word_t* pblk; bool is_gap;
8040  if (blk)
8041  {
8042  pblk = blk;
8043  is_gap = BM_IS_GAP(blk);
8044  }
8045  else
8046  {
8047  pblk = arg_blk;
8048  is_gap = BM_IS_GAP(arg_blk);
8049  }
8050 
8051  if (is_gap)
8052  {
8053  unsigned found = bm::gap_find_first(BMGAP_PTR(pblk), pos);
8054  if (found)
8055  return true;
8056  }
8057  else
8058  {
8059  unsigned found = bm::bit_find_first(pblk, pos);
8060  if (found)
8061  return true;
8062  }
8063  return false;
8064  }
8065 
8066  bool arg_gap = BM_IS_GAP(arg_blk);
8067  bool gap = BM_IS_GAP(blk);
8068 
8069  if (arg_gap != gap)
8070  {
8071  //BM_DECLARE_TEMP_BLOCK(temp_blk);
8072  bm::bit_block_t temp_blk;
8073  bm::word_t* blk1; bm::word_t* blk2;
8074 
8075  if (gap)
8076  {
8078  BMGAP_PTR(blk));
8079  blk1 = (bm::word_t*)temp_blk;
8080  blk2 = (bm::word_t*)arg_blk;
8081  }
8082  else
8083  {
8085  BMGAP_PTR(arg_blk));
8086  blk1 = (bm::word_t*)blk;
8087  blk2 = (bm::word_t*)temp_blk;
8088  }
8089  bool found = bm::bit_find_first_diff(blk1, blk2, pos);
8090  if (found)
8091  return true;
8092  }
8093  else
8094  {
8095  if (gap)
8096  {
8097  bool found =
8099  BMGAP_PTR(arg_blk), pos);
8100  if (found)
8101  return true;
8102  }
8103  else
8104  {
8105  bool found = bm::bit_find_first_diff(blk, arg_blk, pos);
8106  if (found)
8107  return true;
8108  }
8109  }
8110  return false;
8111 }
8112 
8113 
8114 
8115 
8116 /**
8117  Bit-block get adapter, takes bitblock and represents it as a
8118  get_32() accessor function
8119  \internal
8120 */
8122 {
8123 public:
8124  bitblock_get_adapter(const bm::word_t* bit_block) : b_(bit_block) {}
8125 
8127  bm::word_t get_32() BMNOEXCEPT { return *b_++; }
8128 private:
8129  const bm::word_t* b_;
8130 };
8131 
8132 
8133 /**
8134  Bit-block store adapter, takes bitblock and saves results into it
8135  \internal
8136 */
8138 {
8139 public:
8140  bitblock_store_adapter(bm::word_t* bit_block) : b_(bit_block) {}
8142  void push_back(bm::word_t w) { *b_++ = w; }
8143 private:
8144  bm::word_t* b_;
8145 };
8146 
8147 /**
8148  Bit-block sum adapter, takes values and sums it
8149  /internal
8150 */
8152 {
8153 public:
8154  bitblock_sum_adapter() : sum_(0) {}
8156  void push_back(bm::word_t w) BMNOEXCEPT { this->sum_+= w; }
8157  /// Get accumulated sum
8158  bm::word_t sum() const BMNOEXCEPT { return this->sum_; }
8159 private:
8160  bm::word_t sum_;
8161 };
8162 
8163 /**
8164  Adapter to get words from a range stream
8165  (see range serialized bit-block)
8166  \internal
8167 */
8168 template<class DEC> class decoder_range_adapter
8169 {
8170 public:
8171  decoder_range_adapter(DEC& dec, unsigned from_idx, unsigned to_idx)
8172  : decoder_(dec),
8173  from_(from_idx),
8174  to_(to_idx),
8175  cnt_(0)
8176  {}
8177 
8179  {
8180  if (cnt_ < from_ || cnt_ > to_)
8181  {
8182  ++cnt_; return 0;
8183  }
8184  ++cnt_;
8185  return decoder_.get_32();
8186  }
8187 
8188 private:
8189  DEC& decoder_;
8190  unsigned from_;
8191  unsigned to_;
8192  unsigned cnt_;
8193 };
8194 
8195 
8196 /*!
8197  Abstract recombination algorithm for two bit-blocks
8198  Bit blocks can come as dserialization decoders or bit-streams
8199 */
8200 template<class It1, class It2, class BinaryOp, class Encoder>
8201 void bit_recomb(It1& it1, It2& it2,
8202  BinaryOp& op,
8203  Encoder& enc,
8204  unsigned block_size = bm::set_block_size) BMNOEXCEPT
8205 {
8206  for (unsigned i = 0; i < block_size; ++i)
8207  {
8208  bm::word_t w1 = it1.get_32();
8209  bm::word_t w2 = it2.get_32();
8210  bm::word_t w = op(w1, w2);
8211  enc.push_back( w );
8212  } // for
8213 }
8214 
8215 /// Bit AND functor
8216 template<typename W> struct bit_AND
8217 {
8218  W operator()(W w1, W w2) BMNOEXCEPT { return w1 & w2; }
8219 };
8220 
8221 /// Bit OR functor
8222 template<typename W> struct bit_OR
8223 {
8224  W operator()(W w1, W w2) BMNOEXCEPT { return w1 | w2; }
8225 };
8226 
8227 /// Bit SUB functor
8228 template<typename W> struct bit_SUB
8229 {
8230  W operator()(W w1, W w2) BMNOEXCEPT { return w1 & ~w2; }
8231 };
8232 
8233 /// Bit XOR functor
8234 template<typename W> struct bit_XOR
8235 {
8236  W operator()(W w1, W w2) BMNOEXCEPT { return w1 ^ w2; }
8237 };
8238 
8239 /// Bit ASSIGN functor
8240 template<typename W> struct bit_ASSIGN
8241 {
8242  W operator()(W, W w2) BMNOEXCEPT { return w2; }
8243 };
8244 
8245 /// Bit COUNT functor
8246 template<typename W> struct bit_COUNT
8247 {
8248  W operator()(W w1, W w2) BMNOEXCEPT
8249  {
8250  w1 = 0;
8251  BM_INCWORD_BITCOUNT(w1, w2);
8252  return w1;
8253  }
8254 };
8255 
8256 /// Bit COUNT AND functor
8257 template<typename W> struct bit_COUNT_AND
8258 {
8259  W operator()(W w1, W w2) BMNOEXCEPT
8260  {
8261  W r = 0;
8262  BM_INCWORD_BITCOUNT(r, w1 & w2);
8263  return r;
8264  }
8265 };
8266 
8267 /// Bit COUNT XOR functor
8268 template<typename W> struct bit_COUNT_XOR
8269 {
8270  W operator()(W w1, W w2) BMNOEXCEPT
8271  {
8272  W r = 0;
8273  BM_INCWORD_BITCOUNT(r, w1 ^ w2);
8274  return r;
8275  }
8276 };
8277 
8278 /// Bit COUNT OR functor
8279 template<typename W> struct bit_COUNT_OR
8280 {
8281  W operator()(W w1, W w2) BMNOEXCEPT
8282  {
8283  W r = 0;
8284  BM_INCWORD_BITCOUNT(r, w1 | w2);
8285  return r;
8286  }
8287 };
8288 
8289 
8290 /// Bit COUNT SUB AB functor
8291 template<typename W> struct bit_COUNT_SUB_AB
8292 {
8293  W operator()(W w1, W w2) BMNOEXCEPT
8294  {
8295  W r = 0;
8296  BM_INCWORD_BITCOUNT(r, w1 & (~w2));
8297  return r;
8298  }
8299 };
8300 
8301 /// Bit SUB BA functor
8302 template<typename W> struct bit_COUNT_SUB_BA
8303 {
8304  W operator()(W w1, W w2) BMNOEXCEPT
8305  {
8306  W r = 0;
8307  BM_INCWORD_BITCOUNT(r, w2 & (~w1));
8308  return r;
8309  }
8310 };
8311 
8312 /// Bit COUNT A functor
8313 template<typename W> struct bit_COUNT_A
8314 {
8316  {
8317  W r = 0;
8318  BM_INCWORD_BITCOUNT(r, w1);
8319  return r;
8320  }
8321 };
8322 
8323 /// Bit COUNT B functor
8324 template<typename W> struct bit_COUNT_B
8325 {
8327  {
8328  W r = 0;
8329  BM_INCWORD_BITCOUNT(r, w2);
8330  return r;
8331  }
8332 };
8333 
8334 typedef
8336  const gap_word_t*);
8337 
8338 typedef
8339 gap_word_t* (*gap_operation_func_type)(const gap_word_t* BMRESTRICT,
8340  const gap_word_t* BMRESTRICT,
8342  unsigned& );
8343 
8344 typedef
8346  const bm::word_t* BMRESTRICT);
8347 
8348 
8349 template<bool T>
8351 {
8352  static
8354  static
8356  static
8358 
8359  static
8361  {
8362  return gap2bit_table_[i];
8363  }
8364 
8365  static
8367  {
8368  return gapop_table_[i];
8369  }
8370 
8371  static
8373  {
8374  return bit_op_count_table_[i];
8375  }
8376 };
8377 
8378 template<bool T>
8381  &gap_and_to_bitset<bm::gap_word_t>, // set_AND
8382  &gap_add_to_bitset<bm::gap_word_t>, // set_OR
8383  &gap_sub_to_bitset<bm::gap_word_t>, // set_SUB
8384  &gap_xor_to_bitset<bm::gap_word_t>, // set_XOR
8385  0
8386 };
8387 
8388 template<bool T>
8391  &gap_operation_and, // set_AND
8392  &gap_operation_or, // set_OR
8393  &gap_operation_sub, // set_SUB
8394  &gap_operation_xor, // set_XOR
8395  0
8396 };
8397 
8398 
8399 template<bool T>
8402  0, // set_AND
8403  0, // set_OR
8404  0, // set_SUB
8405  0, // set_XOR
8406  0, // set_ASSIGN
8407  0, // set_COUNT
8408  &bit_operation_and_count, // set_COUNT_AND
8409  &bit_operation_xor_count, // set_COUNT_XOR
8410  &bit_operation_or_count, // set_COUNT_OR
8411  &bit_operation_sub_count, // set_COUNT_SUB_AB
8412  &bit_operation_sub_count_inv, // set_COUNT_SUB_BA
8413  0, // set_COUNT_A
8414  0, // set_COUNT_B
8415 };
8416 
8417 /**
8418  Size of bit decode wave in words
8419  @internal
8420  */
8421 const unsigned short set_bitscan_wave_size = 4;
8422 /*!
8423  \brief Unpacks word wave (Nx 32-bit words)
8424  \param w_ptr - pointer on wave start
8425  \param bits - pointer on the result array
8426  \return number of bits in the list
8427 
8428  @ingroup bitfunc
8429  @internal
8430 */
8431 inline
8432 unsigned short
8434  unsigned char* BMRESTRICT bits) BMNOEXCEPT
8435 {
8436  bm::word_t w0, w1;
8437  unsigned short cnt0;
8438 
8439  w0 = w_ptr[0];
8440  w1 = w_ptr[1];
8441 
8442 #if defined(BMAVX512OPT) || defined(BMAVX2OPT) || defined(BMSSE42OPT)
8443  // combine into 64-bit word and scan (when HW popcnt64 is available)
8444  bm::id64_t w = (bm::id64_t(w1) << 32) | w0;
8445  cnt0 = (unsigned short) bm::bitscan_popcnt64(w, bits);
8446 
8447  w0 = w_ptr[2];
8448  w1 = w_ptr[3];
8449  w = (bm::id64_t(w1) << 32) | w0;
8450  cnt0 += bm::bitscan_popcnt64(w, bits + cnt0, 64);
8451 #else
8452  // decode wave as two 32-bit bitscan decodes
8453  cnt0 = bm::bitscan_popcnt(w0, bits);
8454  cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 32);
8455 
8456  w0 = w_ptr[2];
8457  w1 = w_ptr[3];
8458  cnt0 += bm::bitscan_popcnt(w0, bits + cnt0, 64);
8459  cnt0 += bm::bitscan_popcnt(w1, bits + cnt0, 64+32);
8460 #endif
8461  return cnt0;
8462 }
8463 
8464 #if defined (BM64_SSE4) || defined(BM64_AVX2) || defined(BM64_AVX512)
8465 /**
8466  bit index to word gather-scatter algorithm (SIMD)
8467  @ingroup bitfunc
8468  @internal
8469 */
8470 inline
8472  const bm::word_t* BMRESTRICT blk,
8473  const unsigned* BMRESTRICT idx,
8474  unsigned size, unsigned start,
8475  unsigned bit_idx) BMNOEXCEPT
8476 {
8477 typedef unsigned TRGW;
8478 typedef unsigned IDX;
8479 #if defined(BM64_SSE4)
8480  // optimized for unsigned
8481  if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8482  {
8483  sse4_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8484  return;
8485  }
8486 #elif defined(BM64_AVX2) || defined(BM64_AVX512)
8487  if (bm::conditional<sizeof(TRGW)==4 && sizeof(IDX)==4>::test())
8488  {
8489  avx2_bit_block_gather_scatter(arr, blk, idx, size, start, bit_idx);
8490  return;
8491  }
8492 #else
8493  BM_ASSERT(0);
8494 #endif
8495 }
8496 #endif
8497 
8498 /**
8499  bit index to word gather-scatter algorithm
8500  @ingroup bitfunc
8501  @internal
8502 */
8503 template<typename TRGW, typename IDX, typename SZ>
8505  const bm::word_t* BMRESTRICT blk,
8506  const IDX* BMRESTRICT idx,
8507  SZ size, SZ start, unsigned bit_idx) BMNOEXCEPT
8508 {
8509  // TODO: SIMD for 64-bit index sizes and 64-bit target value size
8510  //
8511  TRGW mask1 = 1;
8512  const SZ len = (size - start);
8513  const SZ len_unr = len - (len % 2);
8514  SZ k = 0;
8515  for (; k < len_unr; k+=2)
8516  {
8517  const SZ base = start + k;
8518  const unsigned nbitA = unsigned(idx[base] & bm::set_block_mask);
8519  arr[base] |= (TRGW(bool(blk[nbitA >> bm::set_word_shift] &
8520  (mask1 << (nbitA & bm::set_word_mask)))) << bit_idx);
8521  const unsigned nbitB = unsigned(idx[base + 1] & bm::set_block_mask);
8522  arr[base+1] |= (TRGW(bool(blk[nbitB >> bm::set_word_shift] &
8523  (mask1 << (nbitB & bm::set_word_mask)))) << bit_idx);
8524  } // for k
8525  for (; k < len; ++k)
8526  {
8527  unsigned nbit = unsigned(idx[start + k] & bm::set_block_mask);
8528  arr[start + k] |= (TRGW(bool(blk[nbit >> bm::set_word_shift] &
8529  (mask1 << (nbit & bm::set_word_mask)))) << bit_idx);
8530  } // for k
8531 }
8532 
8533 /**
8534  @brief block boundaries look ahead U32
8535 
8536  @param idx - array to look into
8537  @param size - array size
8538  @param nb - block number to look ahead
8539  @param start - start offset in idx
8540 
8541  @return block boundary offset end (no more match at the returned offset)
8542 
8543  @internal
8544 */
8545 inline
8547  bm::id64_t size, bm::id64_t nb, bm::id64_t start) BMNOEXCEPT
8548 {
8549  BM_ASSERT(idx);
8550  BM_ASSERT(start < size);
8551 
8552  // TODO: SIMD for 64-bit index vector
8553  for (;(start < size) &&
8554  (nb == (idx[start] >> bm::set_block_shift)); ++start)
8555  {}
8556  return start;
8557 }
8558 
8559 /**
8560  @brief block boundaries look ahead U32
8561 
8562  @param idx - array to look into
8563  @param size - array size
8564  @param nb - block number to look ahead
8565  @param start - start offset in idx
8566 
8567  @return block boundary offset end (no more match at the returned offset)
8568 
8569  @internal
8570 */
8571 inline
8572 unsigned idx_arr_block_lookup_u32(const unsigned* idx,
8573  unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
8574 {
8575  BM_ASSERT(idx);
8576  BM_ASSERT(start < size);
8577 
8578 #if defined(VECT_ARR_BLOCK_LOOKUP)
8579  return VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start);
8580 #else
8581  for (;(start < size) &&
8582  (nb == unsigned(idx[start] >> bm::set_block_shift)); ++start)
8583  {}
8584  return start;
8585 #endif
8586 }
8587 
8588 // --------------------------------------------------------------
8589 
8590 
8591 /**
8592  @brief set bits in a bit-block using global index
8593 
8594  @param idx - array to look into
8595  @param block - block pointer to set bits
8596  @param start - index array start
8597  @param stop - index array stop in a range [start..stop)
8598 
8599  @return block boundary offset end (no more match at the returned offset)
8600 
8601  @internal
8602  @ingroup bitfunc
8603 */
8604 inline
8606  const bm::id64_t* BMRESTRICT idx,
8607  bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
8608 {
8609  // TODO: SIMD for 64-bit mode
8610  for (bm::id64_t i = start; i < stop; ++i)
8611  {
8612  bm::id64_t n = idx[i];
8613  unsigned nbit = unsigned(n & bm::set_block_mask);
8614  unsigned nword = nbit >> bm::set_word_shift;
8615  nbit &= bm::set_word_mask;
8616  block[nword] |= (1u << nbit);
8617  } // for i
8618 }
8619 
8620 
8621 /**
8622  @brief set bits in a bit-block using global index
8623 
8624  @param idx - array to look into
8625  @param block - block pointer to set bits
8626  @param start - index array start
8627  @param stop - index array stop in a range [start..stop)
8628 
8629  @return block boundary offset end (no more match at the returned offset)
8630 
8631  @internal
8632  @ingroup bitfunc
8633 */
8634 inline
8636  const unsigned* BMRESTRICT idx,
8637  unsigned start, unsigned stop ) BMNOEXCEPT
8638 {
8639 #if defined(VECT_SET_BLOCK_BITS)
8640  VECT_SET_BLOCK_BITS(block, idx, start, stop);
8641 #else
8642  for (unsigned i = start; i < stop; ++i)
8643  {
8644  unsigned n = idx[i];
8645  unsigned nbit = unsigned(n & bm::set_block_mask);
8646  unsigned nword = nbit >> bm::set_word_shift;
8647  nbit &= bm::set_word_mask;
8648  block[nword] |= (1u << nbit);
8649  } // for i
8650 #endif
8651 }
8652 
8653 
8654 
8655 // --------------------------------------------------------------
8656 
8657 /**
8658  @brief array range detector
8659  @internal
8660 */
8661 inline
8663  unsigned& left, unsigned& right) BMNOEXCEPT
8664 {
8665  BM_ASSERT(arr);
8666 
8667  unsigned i, j;
8668  for (i = 0; i < bm::set_sub_array_size; ++i)
8669  {
8670  if (arr[i])
8671  {
8672  left = i;
8673  break;
8674  }
8675  }
8676  if (i == bm::set_sub_array_size)
8677  {
8678  left = right = 0;
8679  return false; // nothing here
8680  }
8681  for (j = bm::set_sub_array_size-1; j != i; --j)
8682  {
8683  if (arr[j])
8684  break;
8685  }
8686  right = j;
8687  return true;
8688 }
8689 
8690 // --------------------------------------------------------------
8691 
8692 /**
8693  Linear lower bound search in unsigned array
8694  @internal
8695 */
8696 inline
8697 unsigned lower_bound_linear_u32(const unsigned* arr, unsigned target,
8698  unsigned from, unsigned to) BMNOEXCEPT
8699 {
8700  BM_ASSERT(arr);
8701  BM_ASSERT(from <= to);
8702 
8703 #if defined(VECT_LOWER_BOUND_SCAN_U32)
8704  return VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to);
8705 #else
8706  for (; from <= to; ++from)
8707  {
8708  if (arr[from] >= target)
8709  break;
8710  }
8711  return from;
8712 #endif
8713 }
8714 
8715 /**
8716  Linear lower bound search in unsigned LONG array
8717  @internal
8718 */
8719 inline
8720 unsigned lower_bound_linear_u64(const unsigned long long* arr,
8721  unsigned long long target,
8722  unsigned from, unsigned to) BMNOEXCEPT
8723 {
8724  BM_ASSERT(arr);
8725  BM_ASSERT(from <= to);
8726 
8727  // TODO: implement vectorized scan on u64 ints
8728  for (; from <= to; ++from)
8729  {
8730  if (arr[from] >= target)
8731  break;
8732  }
8733  return from;
8734 }
8735 
8736 
8737 
8738 // --------------------------------------------------------------
8739 
8740 /**
8741  Hybrid, binary-linear lower bound search in unsigned array
8742  @internal
8743 */
8744 inline
8745 unsigned lower_bound_u32(const unsigned* arr, unsigned target,
8746  unsigned from, unsigned to) BMNOEXCEPT
8747 {
8748  BM_ASSERT(arr);
8749  BM_ASSERT(from <= to);
8750  const unsigned linear_cutoff = 32;
8751 
8752  unsigned l = from; unsigned r = to;
8753  unsigned dist = r - l;
8754  if (dist < linear_cutoff)
8755  {
8756  return bm::lower_bound_linear_u32(arr, target, l, r);
8757  }
8758 
8759  while (l <= r)
8760  {
8761  unsigned mid = (r-l)/2+l;
8762  if (arr[mid] < target)
8763  l = mid+1;
8764  else
8765  r = mid-1;
8766  dist = r - l;
8767  if (dist < linear_cutoff)
8768  {
8769  return bm::lower_bound_linear_u32(arr, target, l, r);
8770  }
8771  }
8772  return l;
8773 }
8774 
8775 /**
8776  Hybrid, binary-linear lower bound search in unsigned LONG array
8777  @internal
8778 */
8779 inline
8780 unsigned lower_bound_u64(const unsigned long long* arr,
8781  unsigned long long target,
8782  unsigned from, unsigned to) BMNOEXCEPT
8783 {
8784  BM_ASSERT(arr);
8785  BM_ASSERT(from <= to);
8786  const unsigned linear_cutoff = 32;
8787 
8788  unsigned l = from; unsigned r = to;
8789  unsigned dist = r - l;
8790  if (dist < linear_cutoff)
8791  {
8792  return bm::lower_bound_linear_u64(arr, target, l, r);
8793  }
8794 
8795  while (l <= r)
8796  {
8797  unsigned mid = (r - l) / 2 + l;
8798  if (arr[mid] < target)
8799  l = mid + 1;
8800  else
8801  r = mid - 1;
8802  dist = r - l;
8803  if (dist < linear_cutoff)
8804  {
8805  return bm::lower_bound_linear_u64(arr, target, l, r);
8806  }
8807  }
8808  return l;
8809 }
8810 
8811 
8812 /**
8813  calculate bvector<> global bit-index from block-local coords
8814  @return bit index in linear bit-vector coordinates
8815  @internal
8816 */
8817 #ifdef BM64ADDR
8818 inline
8819 bm::id64_t block_to_global_index(unsigned i, unsigned j,
8820  unsigned block_idx) BMNOEXCEPT
8821 {
8823  base_idx += j * bm::gap_max_bits;
8824  return block_idx + base_idx;
8825 }
8826 #else
8827 inline
8828 bm::id_t block_to_global_index(unsigned i, unsigned j,
8829  unsigned block_idx) BMNOEXCEPT
8830 {
8831  unsigned base_idx = i * bm::set_sub_array_size * bm::gap_max_bits;
8832  base_idx += j * bm::gap_max_bits;
8833  return block_idx + base_idx;
8834 }
8835 #endif
8836 
8837 // --------------------------------------------------------------
8838 // Functions to work with int values stored in 64-bit pointers
8839 // --------------------------------------------------------------
8840 
8841 /*!
8842  \brief helper union to interpret pointer as integers
8843  @internal
8844 */
8846 {
8849  unsigned short i16[4];
8850 };
8851 
8852 /*!
8853  Test presense of value in payload pointer
8854  @internal
8855 */
8856 inline
8858 {
8859  if (v == 0)
8860  {
8861  return (ptr.i16[1] == 0);
8862  }
8863  bm::id64_t r = (ptr.i16[1] == v) | (ptr.i16[2] == v) | (ptr.i16[3] == v);
8864  return r;
8865 }
8866 
8867 // --------------------------------------------------------------------------
8868 
8869 } // namespace bm
8870 
8871 #endif
bm::gap_shift_l1
bool gap_shift_l1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Left shift GAP block by 1 bit.
Definition: bmfunc.h:2946
bm::gap_find_interval_start
bool gap_find_interval_start(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a GAP block.
Definition: bmfunc.h:2016
bm::bit_andnot_arr_ffmask
void bit_andnot_arr_ffmask(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND NOT with constant ~0 mask operation.
Definition: bmfunc.h:7276
bm::operation_functions::gap_operation
static gap_operation_func_type gap_operation(unsigned i)
Definition: bmfunc.h:8366
BM_VECT_ALIGN
#define BM_VECT_ALIGN
Definition: bmdef.h:359
bm::block_find_interval_start
unsigned block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find start of the current 111 interval.
Definition: bmfunc.h:5528
bm::find_not_null_ptr
bool find_not_null_ptr(bm::word_t ***arr, N start, N size, N *pos) BMNOEXCEPT
Definition: bmfunc.h:923
bm::bit_block_change_bc
void bit_block_change_bc(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT gc, unsigned *BMRESTRICT bc) BMNOEXCEPT
Definition: bmfunc.h:4581
bm::bmi_blsi_u64
BMFORCEINLINE unsigned long long bmi_blsi_u64(unsigned long long w)
Definition: bmutil.h:342
VECT_BITCOUNT_OR
#define VECT_BITCOUNT_OR(first, last, mask)
Definition: bmsse4.h:1725
bm::gap_count_or
BMFORCEINLINE unsigned gap_count_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount OR operation test.
Definition: bmfunc.h:5810
bm::bit_block_sub_count
unsigned bit_block_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6319
bm::bit_block_sub
bm::id64_t bit_block_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock SUB (AND NOT) operation. Function does not analyse availability of source and destinat...
Definition: bmfunc.h:7019
bm::bit_block_xor_2way
bm::id64_t bit_block_xor_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 ^ source2) bitblock XOR operation.
Definition: bmfunc.h:6825
bm::bit_is_all_zero
bool bit_is_all_zero(const bm::word_t *BMRESTRICT start) BMNOEXCEPT
Returns "true" if all bits in the block are 0.
Definition: bmfunc.h:1045
bm::sse42_gap_test
unsigned sse42_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition: bmsse4.h:1106
VECT_BLOCK_CHANGE_BC
#define VECT_BLOCK_CHANGE_BC(block, gc, bc)
Definition: bmsse4.h:1822
bm::bit_block_t::bunion_t
Definition: bmutil.h:58
IS_EMPTY_BLOCK
#define IS_EMPTY_BLOCK(addr)
Definition: bmdef.h:154
bm::bit_block_count
bm::id_t bit_block_count(const bm::word_t *block) BMNOEXCEPT
Bitcount for bit block.
Definition: bmfunc.h:4379
bm::d_copy_func
d-Gap copy functor
Definition: bmfunc.h:2166
bm::bv_statistics::max_serialize_mem
size_t max_serialize_mem
estimated maximum memory for serialization
Definition: bmfunc.h:60
bm::bit_decode_cache
bit-decode cache structure
Definition: bmfunc.h:132
bm::bv_statistics::gaps_by_level
unsigned long long gaps_by_level[bm::gap_levels]
number of GAP blocks at each level
Definition: bmfunc.h:64
bm::gap_operation_sub
gap_word_t * gap_operation_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP SUB (AND NOT) operation.
Definition: bmfunc.h:5835
bm::gap_bitset_or_count
bm::id_t gap_bitset_or_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block OR masked by GAP block.
Definition: bmfunc.h:3790
BM_IS_GAP
#define BM_IS_GAP(ptr)
Definition: bmdef.h:181
bm::block_to_global_index
bm::id_t block_to_global_index(unsigned i, unsigned j, unsigned block_idx) BMNOEXCEPT
calculate bvector<> global bit-index from block-local coords
Definition: bmfunc.h:8828
bm::gapcmp
int gapcmp(const T *buf1, const T *buf2) BMNOEXCEPT
Lexicographical comparison of GAP buffers.
Definition: bmfunc.h:2256
bm::copy_to_array_functor::copy_to_array_functor
copy_to_array_functor(B *bits)
Definition: bmfunc.h:389
bm::lower_bound_linear_u32
unsigned lower_bound_linear_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned array.
Definition: bmfunc.h:8697
bm::bit_operation_sub_count_inv
bm::id_t bit_operation_sub_count_inv(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs inverted bitblock SUB operation and calculates bitcount of the result.
Definition: bmfunc.h:6626
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx) BMNOEXCEPT
Definition: bmfunc.h:394
bm::set_block_bits_u32
void set_block_bits_u32(bm::word_t *BMRESTRICT block, const unsigned *BMRESTRICT idx, unsigned start, unsigned stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition: bmfunc.h:8635
bm::set_block_size
const unsigned set_block_size
Definition: bmconst.h:54
bm::gap_operation_func_type
gap_word_t *(* gap_operation_func_type)(const gap_word_t *BMRESTRICT, const gap_word_t *BMRESTRICT, gap_word_t *BMRESTRICT, unsigned &)
Definition: bmfunc.h:8339
bm::gap_control_sum
unsigned gap_control_sum(const T *buf) BMNOEXCEPT
Calculates sum of all words in GAP block. (For debugging purposes)
Definition: bmfunc.h:3902
bm::gap_any_range
bool gap_any_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range.
Definition: bmfunc.h:1933
bm::pair::first
First first
Definition: bmfunc.h:123
bm::ptr_payload_t
helper union to interpret pointer as integers
Definition: bmfunc.h:8845
bm::bit_block_find_interval_end
bool bit_block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a BIT block.
Definition: bmfunc.h:5373
bm::gap_xor_to_bitset
void gap_xor_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
XOR GAP block to bitblock.
Definition: bmfunc.h:3408
bm::set_gap_level
void set_gap_level(T *buf, int level) BMNOEXCEPT
Sets GAP block capacity level.
Definition: bmfunc.h:4020
bm::bitblock_sum_adapter::bitblock_sum_adapter
bitblock_sum_adapter()
Definition: bmfunc.h:8154
bm::count_trailing_zeros_u64
unsigned count_trailing_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan fwd
Definition: bmutil.h:381
bm::bit_find_first
bool bit_find_first(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT pos) BMNOEXCEPT
BIT block find the first set bit.
Definition: bmfunc.h:7552
bm::bit_scan_forward32
unsigned bit_scan_forward32(unsigned value) BMNOEXCEPT
Definition: bmutil.h:316
VECT_SUB_BLOCK
#define VECT_SUB_BLOCK(dst, src)
Definition: bmsse4.h:1761
bm::bit_operation_xor_any
bm::id_t bit_operation_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation test.
Definition: bmfunc.h:7381
bm::set_AND
Definition: bmconst.h:154
bm::bits_in_block
const unsigned bits_in_block
Definition: bmconst.h:113
bm::bit_COUNT::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8248
bm::gap_count_xor
BMFORCEINLINE unsigned gap_count_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount XOR operation test.
Definition: bmfunc.h:5765
bm::bit_operation_and_count
bm::id_t bit_operation_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation and calculates bitcount of the result.
Definition: bmfunc.h:6540
bm::block_find_rank
SIZE_TYPE block_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
Find rank in block (GAP or BIT)
Definition: bmfunc.h:7761
bm::set_SUB
Definition: bmconst.h:156
bm::set_block_bits_u64
void set_block_bits_u64(bm::word_t *BMRESTRICT block, const bm::id64_t *BMRESTRICT idx, bm::id64_t start, bm::id64_t stop) BMNOEXCEPT
set bits in a bit-block using global index
Definition: bmfunc.h:8605
bm::bit_block_change32
unsigned bit_block_change32(const bm::word_t *block, unsigned size) BMNOEXCEPT
Definition: bmfunc.h:4533
bm::gap_2_dgap
T * gap_2_dgap(const T *BMRESTRICT gap_buf, T *BMRESTRICT dgap_buf, bool copy_head=true) BMNOEXCEPT
Convert GAP buffer into D-GAP buffer.
Definition: bmfunc.h:2188
bm::set_COUNT
Definition: bmconst.h:159
bm::bv_statistics::bit_blocks
size_t bit_blocks
Number of bit blocks.
Definition: bmfunc.h:56
bm::bit_block_any_range
bm::id_t bit_block_any_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
bm::bit_invert
void bit_invert(T *start) BMNOEXCEPT
Definition: bmfunc.h:5253
bm::set_sub_total_bits
const unsigned set_sub_total_bits
Definition: bmconst.h:99
bm::all_set
Structure carries pointer on bit block with all bits 1.
Definition: bmfunc.h:842
bm::set_representation
set_representation
set representation variants
Definition: bmconst.h:201
bm::for_each_nzblock_range
void for_each_nzblock_range(T ***root, N top_size, N nb_from, N nb_to, F &f) BMNOEXCEPT
Definition: bmfunc.h:1325
VECT_BITCOUNT
#define VECT_BITCOUNT(first, last)
Definition: bmsse4.h:1719
bm::gap_convert_to_arr
D gap_convert_to_arr(D *BMRESTRICT dest, const T *BMRESTRICT buf, unsigned dest_len, bool invert=false) BMNOEXCEPT
Convert gap block into array of ints corresponding to 1 bits.
Definition: bmfunc.h:4320
bm::is_bits_one
bool is_bits_one(const bm::wordop_t *start) BMNOEXCEPT
Returns "true" if all bits in the block are 1.
Definition: bmfunc.h:5277
VECT_SHIFT_R1_AND
#define VECT_SHIFT_R1_AND(b, co, m, digest)
Definition: bmsse4.h:1806
bm::gap_free_elements
unsigned gap_free_elements(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returns number of free elements in GAP block array. Difference between GAP block capacity on this lev...
Definition: bmfunc.h:4063
VECT_BIT_FIND_DIFF
#define VECT_BIT_FIND_DIFF(src1, src2, pos)
Definition: bmsse4.h:1829
VECT_SHIFT_R1
#define VECT_SHIFT_R1(b, acc, co)
Definition: bmsse4.h:1803
bm::for_each_nzblock_if
bool for_each_nzblock_if(T ***root, BI size1, F &f) BMNOEXCEPT
Definition: bmfunc.h:1613
bm::bit_count_table
Structure to aid in counting bits table contains count of bits in 0-255 diapason of numbers.
Definition: bmconst.h:287
bm::gap_operation_any_and
BMFORCEINLINE unsigned gap_operation_any_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP AND operation test.
Definition: bmfunc.h:5670
BMGAP_PTR
#define BMGAP_PTR(ptr)
Definition: bmdef.h:179
BM_INCWORD_BITCOUNT
#define BM_INCWORD_BITCOUNT(cnt, w)
Definition: bmdef.h:386
bm::bit_OR
Bit OR functor.
Definition: bmfunc.h:8222
bm::ptrp_test
bm::id64_t ptrp_test(ptr_payload_t ptr, bm::gap_word_t v) BMNOEXCEPT
Definition: bmfunc.h:8857
bm::bit_count_nonzero_size
unsigned bit_count_nonzero_size(const T *blk, unsigned data_size) BMNOEXCEPT
Inspects block for full zero words.
Definition: bmfunc.h:7407
bm::block_is_interval
bool block_is_interval(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] and border bits are 0.
Definition: bmfunc.h:5324
bm::gap_count_and
unsigned gap_count_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount AND operation test.
Definition: bmfunc.h:5687
VECT_XOR_BLOCK_2WAY
#define VECT_XOR_BLOCK_2WAY(dst, src1, src2)
Definition: bmsse4.h:1773
bm::bit_COUNT_AND::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8259
VECT_OR_BLOCK
#define VECT_OR_BLOCK(dst, src)
Definition: bmsse4.h:1749
bm::gap_capacity
unsigned gap_capacity(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity.
Definition: bmfunc.h:1114
bm::get_block_coord
void get_block_coord(BI_TYPE nb, unsigned &i, unsigned &j) BMNOEXCEPT
Recalc linear bvector block index into 2D matrix coordinates.
Definition: bmfunc.h:147
bm::gap_operation_to_bitset_func_type
void(* gap_operation_to_bitset_func_type)(unsigned *, const gap_word_t *)
Definition: bmfunc.h:8335
bm::bit_block_t
bit-block array wrapped into union for correct interpretation of 32-bit vs 64-bit access vs SIMD
Definition: bmutil.h:56
bm::set_block_digest_pos_shift
const unsigned set_block_digest_pos_shift
Definition: bmconst.h:67
bm::widx_to_digest_mask
BMFORCEINLINE bm::id64_t widx_to_digest_mask(unsigned w_idx) BMNOEXCEPT
Compute digest mask for word address in block.
Definition: bmfunc.h:650
bm::id64_t
unsigned long long int id64_t
Definition: bmconst.h:34
bm::decoder_range_adapter::decoder_range_adapter
decoder_range_adapter(DEC &dec, unsigned from_idx, unsigned to_idx)
Definition: bmfunc.h:8171
bm::bit_block_shift_l1_unr
bool bit_block_shift_l1_unr(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift of bit-block by 1 bit (loop unrolled)
Definition: bmfunc.h:5010
bm::gap_calc_level
int gap_calc_level(unsigned len, const T *glevel_len) BMNOEXCEPT
Calculates GAP block capacity level.
Definition: bmfunc.h:4042
bm::gap_invert
void gap_invert(T *buf) BMNOEXCEPT
Inverts all bits in the GAP buffer.
Definition: bmfunc.h:4001
bm::bit_operation_and_any
bm::id_t bit_operation_and_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock AND operation test.
Definition: bmfunc.h:6564
bm::gap_sub_to_bitset
void gap_sub_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
SUB (AND NOT) GAP block to bitblock.
Definition: bmfunc.h:3320
bm::gap_find_rank
SIZE_TYPE gap_find_rank(const T *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
GAP block find position for the rank.
Definition: bmfunc.h:2049
bm::word_bitcount64
BMFORCEINLINE unsigned word_bitcount64(bm::id64_t x) BMNOEXCEPT
Definition: bmfunc.h:230
bm::all_set::_block
static all_set_block _block
Definition: bmfunc.h:901
VECT_ANDNOT_ARR_2_MASK
#define VECT_ANDNOT_ARR_2_MASK(dst, src, src_end, mask)
Definition: bmsse4.h:1716
bm::gap_bitset_xor_count
bm::id_t gap_bitset_xor_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block XOR masked by GAP block.
Definition: bmfunc.h:3714
bm::gap_convert_to_bitset_smart
unsigned * gap_convert_to_bitset_smart(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf, id_t set_max) BMNOEXCEPT
Smart GAP block to bitblock conversion.
Definition: bmfunc.h:3881
bm::set_word_shift
const unsigned set_word_shift
Definition: bmconst.h:71
BMNOEXCEPT2
#define BMNOEXCEPT2
Definition: bmdef.h:99
bm::sub_op
BMFORCEINLINE unsigned sub_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition: bmfunc.h:5623
bm::gap_and_to_bitset
void gap_and_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr) BMNOEXCEPT
ANDs GAP block to bitblock.
Definition: bmfunc.h:3486
bm::set_sub_array_size
const unsigned set_sub_array_size
Definition: bmconst.h:94
bm::gap_find_first
unsigned gap_find_first(const T *BMRESTRICT buf, unsigned *BMRESTRICT first) BMNOEXCEPT
GAP block find the first set bit.
Definition: bmfunc.h:1193
bm::wordop_t
id64_t wordop_t
Definition: bmconst.h:127
bm::bit_ASSIGN::operator()
W operator()(W, W w2) BMNOEXCEPT
Definition: bmfunc.h:8242
bm::digest_mask
BMFORCEINLINE bm::id64_t digest_mask(unsigned from, unsigned to) BMNOEXCEPT
Compute digest mask for [from..to] positions.
Definition: bmfunc.h:665
bm::check_block_one
bool check_block_one(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks if block has only 1 bits.
Definition: bmfunc.h:7882
bm::bit_block_insert
bm::word_t bit_block_insert(bm::word_t *BMRESTRICT block, unsigned bitpos, bool value) BMNOEXCEPT
insert bit into position and shift the rest right with carryover
Definition: bmfunc.h:4876
bm::xor_op
BMFORCEINLINE unsigned xor_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP xor functor.
Definition: bmfunc.h:5610
bm::bit_XOR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8236
bm::bv_statistics::gap_blocks
size_t gap_blocks
Number of GAP blocks.
Definition: bmfunc.h:57
bm::gap_overhead
unsigned gap_overhead(const T *length, const T *length_end, const T *glevel_len) BMNOEXCEPT
Calculates memory overhead for number of gap blocks sharing the same memory allocation table (level l...
Definition: bmfunc.h:7905
bm::bitcmp
int bitcmp(const T *buf1, const T *buf2, unsigned len) BMNOEXCEPT
Lexicographical comparison of BIT buffers.
Definition: bmfunc.h:4081
bm::bitblock_get_adapter::bitblock_get_adapter
bitblock_get_adapter(const bm::word_t *bit_block)
Definition: bmfunc.h:8124
bm::all_set::all_set_block::all_set_block
all_set_block()
Definition: bmfunc.h:850
bm::all_set::is_full_block
static BMFORCEINLINE bool is_full_block(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:894
bm::sub_bit_block
void sub_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
SUB (AND NOT) bit interval to 1 in the bitblock.
Definition: bmfunc.h:3217
bm::bit_decode_cache::bits
unsigned short bits[65]
Definition: bmfunc.h:134
bm::bitscan
unsigned short bitscan(V w, B *bits) BMNOEXCEPT
Definition: bmfunc.h:554
bm::bv_statistics::gap_cap_overhead
size_t gap_cap_overhead
gap memory overhead between length and capacity
Definition: bmfunc.h:62
bm::set_bitscan_wave_size
const unsigned short set_bitscan_wave_size
Size of bit decode wave in words.
Definition: bmfunc.h:8421
bm::check_block_zero
bool check_block_zero(const bm::word_t *blk, bool deep_scan) BMNOEXCEPT
Checks all conditions and returns true if block consists of only 0 bits.
Definition: bmfunc.h:7858
bm::bit_ASSIGN
Bit ASSIGN functor.
Definition: bmfunc.h:8240
bm::bit_XOR
Bit XOR functor.
Definition: bmfunc.h:8234
bm::sse2_gap_sum_arr
const bm::gap_word_t * sse2_gap_sum_arr(const bm::gap_word_t *BMRESTRICT pbuf, unsigned sse_vect_waves, unsigned *sum)
Gap block population count (array sum) utility.
Definition: bmsse_util.h:891
bm::sum_arr
bm::id64_t sum_arr(const T *first, const T *last) BMNOEXCEPT
Definition: bmfunc.h:1695
bm::word_select64_bitscan
unsigned word_select64_bitscan(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:597
bm::word_select64_linear
unsigned word_select64_linear(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:574
bm::xor_bit_block
void xor_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
XOR bit interval to 1 in the bitblock.
Definition: bmfunc.h:3264
bm::parallel_popcnt_32
int parallel_popcnt_32(unsigned int n) BMNOEXCEPT
Definition: bmfunc.h:215
bm::bit_block_and
bm::id64_t bit_block_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock AND operation. Function does not analyse availability of source and destination blocks...
Definition: bmfunc.h:5940
bm::operation_functions::gap2bit_table_
static gap_operation_to_bitset_func_type gap2bit_table_[bm::set_END]
Definition: bmfunc.h:8353
bm::gap_is_interval
bool gap_is_interval(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if any bits are 1 in GAP buffer in the [left, right] range and flanked with 0s.
Definition: bmfunc.h:1962
bm::bit_block_or
bool bit_block_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock OR operation. Function does not analyse availability of source and destination blocks.
Definition: bmfunc.h:6749
bm::all_bits_mask
const id64_t all_bits_mask
Definition: bmconst.h:128
bm::bit_block_shift_r1_and
bool bit_block_shift_r1_and(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (reference) + AND.
Definition: bmfunc.h:5089
bm::bit_COUNT_XOR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8270
bm::bit_COUNT_SUB_AB::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8293
bm::sse4_bit_block_gather_scatter
void sse4_bit_block_gather_scatter(unsigned *BMRESTRICT arr, const unsigned *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx)
Definition: bmsse4.h:1343
bm::bit_list_4
unsigned bit_list_4(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes (quad-bit based)
Definition: bmfunc.h:456
bm::bitblock_sum_adapter::sum
bm::word_t sum() const BMNOEXCEPT
Get accumulated sum.
Definition: bmfunc.h:8158
bm::gap_add_value
unsigned gap_add_value(T *buf, unsigned pos) BMNOEXCEPT
Add new value to the end of GAP buffer.
Definition: bmfunc.h:2827
bm::bv_statistics::add_bit_block
void add_bit_block() BMNOEXCEPT
cound bit block
Definition: bmfunc.h:67
bm::pair
Pair type.
Definition: bmfunc.h:121
bm::bit_COUNT_SUB_BA
Bit SUB BA functor.
Definition: bmfunc.h:8302
bm::for_each_gap_dbit
void for_each_gap_dbit(const T *buf, F &func)
Iterate gap block as delta-bits with a functor.
Definition: bmfunc.h:4265
bm::set_OR
Definition: bmconst.h:155
bm::idx_arr_block_lookup_u64
bm::id64_t idx_arr_block_lookup_u64(const bm::id64_t *idx, bm::id64_t size, bm::id64_t nb, bm::id64_t start) BMNOEXCEPT
block boundaries look ahead U32
Definition: bmfunc.h:8546
bm::set_block_mask
const unsigned set_block_mask
Definition: bmconst.h:56
bm::bit_block_or_5way
bool bit_block_or_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, const bm::word_t *BMRESTRICT src4) BMNOEXCEPT
5 way (target, source1, source2) bitblock OR operation. Function does not analyse availability of sou...
Definition: bmfunc.h:6910
bm::bit_COUNT_OR
Bit COUNT OR functor.
Definition: bmfunc.h:8279
bm::set_array_mask
const unsigned set_array_mask
Definition: bmconst.h:96
bm::gap_bit_count_unr
unsigned gap_bit_count_unr(const T *buf) BMNOEXCEPT
Calculates number of bits ON in GAP buffer. Loop unrolled version.
Definition: bmfunc.h:1801
VECT_AND_DIGEST_5WAY
#define VECT_AND_DIGEST_5WAY(dst, src1, src2, src3, src4)
Definition: bmsse4.h:1743
bm::block_ptr_array_range
bool block_ptr_array_range(bm::word_t **arr, unsigned &left, unsigned &right) BMNOEXCEPT
array range detector
Definition: bmfunc.h:8662
VECT_GAP_BFIND
#define VECT_GAP_BFIND(buf, pos, is_set)
Definition: bmsse4.h:1835
bm::operation_functions::bit_op_count_table_
static bit_operation_count_func_type bit_op_count_table_[bm::set_END]
Definition: bmfunc.h:8357
bm::bitblock_store_adapter::push_back
BMFORCEINLINE void push_back(bm::word_t w)
Definition: bmfunc.h:8142
bm::gap_split
void gap_split(const T *buf, T *arr0, T *arr1, T &arr0_cnt, T &arr1_cnt) BMNOEXCEPT
Definition: bmfunc.h:1713
bm::xor_swap
void xor_swap(W &x, W &y) BMNOEXCEPT
XOR swap two scalar variables.
Definition: bmfunc.h:909
bm::gap_find_last
unsigned gap_find_last(const T *BMRESTRICT buf, unsigned *BMRESTRICT last) BMNOEXCEPT
GAP block find the last set bit.
Definition: bmfunc.h:1162
bm::bit_block_to_gap
unsigned bit_block_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Converts bit block to GAP.
Definition: bmfunc.h:4163
bm::for_each_dgap
void for_each_dgap(const T *gap_buf, Func &func)
Definition: bmfunc.h:2147
bm::setop2op
bm::operation setop2op(bm::set_operation op) BMNOEXCEPT
Convert set operation to operation.
Definition: bmfunc.h:826
bm::gap_operation_dry_xor
bool gap_operation_dry_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, unsigned &dsize, unsigned limit) BMNOEXCEPT
Definition: bmfunc.h:5725
bm::gap_operation_any_sub
BMFORCEINLINE unsigned gap_operation_any_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP SUB operation test.
Definition: bmfunc.h:5860
bm::gap_set_all
void gap_set_all(T *buf, unsigned set_max, unsigned value) BMNOEXCEPT
Sets all bits to 0 or 1 (GAP)
Definition: bmfunc.h:3933
bm::gap_test
unsigned gap_test(const T *BMRESTRICT buf, unsigned pos) BMNOEXCEPT
Tests if bit = pos is true.
Definition: bmfunc.h:1255
bm::idx_arr_block_lookup_u32
unsigned idx_arr_block_lookup_u32(const unsigned *idx, unsigned size, unsigned nb, unsigned start) BMNOEXCEPT
block boundaries look ahead U32
Definition: bmfunc.h:8572
bm::bit_decode_cache::cvalue
bm::id64_t cvalue
Definition: bmfunc.h:136
bm::gap_block_find
unsigned gap_block_find(const T *BMRESTRICT buf, unsigned nbit, bm::id_t *BMRESTRICT prev) BMNOEXCEPT
Searches for the next 1 bit in the GAP block.
Definition: bmfunc.h:3098
bm::bit_convert_to_arr
T bit_convert_to_arr(T *BMRESTRICT dest, const unsigned *BMRESTRICT src, bm::id_t bits, unsigned dest_len, unsigned mask=0) BMNOEXCEPT
Convert bit block into an array of ints corresponding to 1 bits.
Definition: bmfunc.h:7822
bm::gap_buff_count_op
unsigned gap_buff_count_op(const T *vect1, const T *vect2, F f) BMNOEXCEPT2
Abstract distance(similarity) operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2576
bm::gap_bit_count_to
unsigned gap_bit_count_to(const T *const buf, T right, bool is_corrected=false) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [0, right] range.
Definition: bmfunc.h:2101
bm::gap_buff_any_op
unsigned gap_buff_any_op(const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F f) BMNOEXCEPT2
Abstract distance test operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2507
bm::gap_buff_dry_op
bool gap_buff_dry_op(const T *BMRESTRICT vect1, const T *BMRESTRICT vect2, F &f, unsigned &dlen, unsigned limit) BMNOEXCEPT2
Abstract operation for GAP buffers (predicts legth) Receives functor F as a template argument.
Definition: bmfunc.h:2434
BMNOEXCEPT
#define BMNOEXCEPT
Definition: bmdef.h:79
VECT_SUB_DIGEST_2WAY
#define VECT_SUB_DIGEST_2WAY(dst, src1, src2)
Definition: bmsse4.h:1767
bm::bit_COUNT_B::operator()
W operator()(W, W w2) BMNOEXCEPT
Definition: bmfunc.h:8326
bm::copy_to_array_functor::ptr
B * ptr()
Definition: bmfunc.h:392
bm::bit_operation_or_any
bm::id_t bit_operation_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation test.
Definition: bmfunc.h:6716
bm::wordcmp
int wordcmp(T a, T b)
Lexicographical comparison of two words as bit strings. Auxiliary implementation for testing and refe...
Definition: bmfunc.h:1027
bm::bit_block_gather_scatter
void bit_block_gather_scatter(unsigned *BMRESTRICT arr, const bm::word_t *BMRESTRICT blk, const unsigned *BMRESTRICT idx, unsigned size, unsigned start, unsigned bit_idx) BMNOEXCEPT
bit index to word gather-scatter algorithm (SIMD)
Definition: bmfunc.h:8471
bm::bit_block_or_3way
bool bit_block_or_3way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
3 way (target | source1 | source2) bitblock OR operation. Function does not analyse availability of s...
Definition: bmfunc.h:6866
bm::gap_bit_count_range
unsigned gap_bit_count_range(const T *const buf, unsigned left, unsigned right) BMNOEXCEPT
Counts 1 bits in GAP buffer in the closed [left, right] range.
Definition: bmfunc.h:1867
bm::decoder_range_adapter
Adapter to get words from a range stream (see range serialized bit-block)
Definition: bmfunc.h:8168
VECT_OR_BLOCK_5WAY
#define VECT_OR_BLOCK_5WAY(dst, src1, src2, src3, src4)
Definition: bmsse4.h:1758
bm::set_block_digest_wave_size
const unsigned set_block_digest_wave_size
Definition: bmconst.h:66
bm::for_each_nzblock2
void for_each_nzblock2(T ***root, unsigned size1, F &f)
Definition: bmfunc.h:1491
bm::bitscan_popcnt64
unsigned short bitscan_popcnt64(bm::id64_t w, B *bits) BMNOEXCEPT
Unpacks 64-bit word into list of ON bit indexes using popcnt method.
Definition: bmfunc.h:518
bm::decoder_range_adapter::get_32
bm::word_t get_32() BMNOEXCEPT
Definition: bmfunc.h:8178
bm::gap_level
T gap_level(const T *BMRESTRICT buf) BMNOEXCEPT
Returs GAP blocks capacity level.
Definition: bmfunc.h:1144
VECT_AND_BLOCK
#define VECT_AND_BLOCK(dst, src)
Definition: bmsse4.h:1737
bm::all_set::all_set_block::_p_fullp
bm::word_t * _p_fullp
Definition: bmfunc.h:848
bm::bitblock_get_adapter::get_32
BMFORCEINLINE bm::word_t get_32() BMNOEXCEPT
Definition: bmfunc.h:8127
bm::update_block_digest0
bm::id64_t update_block_digest0(const bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Compute digest for 64 non-zero areas based on existing digest (function revalidates zero areas)
Definition: bmfunc.h:776
bm::pair::pair
pair(First f, Second s)
Definition: bmfunc.h:126
bm::gap_set_value
unsigned gap_set_value(unsigned val, T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Sets or clears bit in the GAP buffer.
Definition: bmfunc.h:2661
bm::calc_block_digest0
bm::id64_t calc_block_digest0(const bm::word_t *const block) BMNOEXCEPT
Compute digest for 64 non-zero areas.
Definition: bmfunc.h:734
FULL_BLOCK_FAKE_ADDR
#define FULL_BLOCK_FAKE_ADDR
Definition: bmdef.h:149
bm::bit_OR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8224
bm::lower_bound_linear_u64
unsigned lower_bound_linear_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Linear lower bound search in unsigned LONG array.
Definition: bmfunc.h:8720
VECT_STREAM_BLOCK
#define VECT_STREAM_BLOCK(dst, src)
Definition: bmsse4.h:1779
bm::bit_operation_xor
bm::word_t * bit_operation_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock XOR operation.
Definition: bmfunc.h:7311
bm::bit_COUNT_B
Bit COUNT B functor.
Definition: bmfunc.h:8324
VECT_BITCOUNT_XOR
#define VECT_BITCOUNT_XOR(first, last, mask)
Definition: bmsse4.h:1728
bm::gap_word_t
unsigned short gap_word_t
Definition: bmconst.h:77
bm::ptr_payload_t::blk
bm::word_t * blk
Definition: bmfunc.h:8847
bm::d_copy_func::operator()
void operator()(T dgap)
Definition: bmfunc.h:2169
bm::set_operation
set_operation
Codes of set operations.
Definition: bmconst.h:152
bm::bit_operation_or
bm::word_t * bit_operation_or(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Block OR operation. Makes analysis if block is 0 or FULL.
Definition: bmfunc.h:6959
bm::all_set::all_set_block::BM_VECT_ALIGN_ATTR
bm::word_t BM_VECT_ALIGN _p[bm::set_block_size] BM_VECT_ALIGN_ATTR
Definition: bmfunc.h:847
bm::test_bit
BMFORCEINLINE unsigned test_bit(const unsigned *block, unsigned bitpos) BMNOEXCEPT
Test 1 bit in a block.
Definition: bmfunc.h:3152
bm::all_set::all_set_block::BM_VECT_ALIGN_ATTR
bm::word_t BM_VECT_ALIGN *_s[bm::set_sub_array_size] BM_VECT_ALIGN_ATTR
Definition: bmfunc.h:846
bm::set_array_shift
const unsigned set_array_shift
Definition: bmconst.h:95
bm::block_any
bool block_any(const bm::word_t *const BMRESTRICT block) BMNOEXCEPT
Returns "true" if one bit is set in the block Function check for block varieties.
Definition: bmfunc.h:5584
bm::bit_decode_cache::bcnt
unsigned bcnt
Definition: bmfunc.h:135
bm::conditional
ad-hoc conditional expressions
Definition: bmutil.h:110
bm::gap_convert_to_bitset
void gap_convert_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT buf) BMNOEXCEPT
GAP block to bitblock conversion.
Definition: bmfunc.h:3859
bm::bit_recomb
void bit_recomb(It1 &it1, It2 &it2, BinaryOp &op, Encoder &enc, unsigned block_size=bm::set_block_size) BMNOEXCEPT
Definition: bmfunc.h:8201
bm::bit_SUB::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8230
bm::bit_list
unsigned bit_list(T w, B *bits) BMNOEXCEPT
Unpacks word into list of ON bit indexes.
Definition: bmfunc.h:438
BM_ASSERT
#define BM_ASSERT
Definition: bmdef.h:130
bm::block_set_table
Structure keeps all-left/right ON bits masks.
Definition: bmconst.h:342
bm::get_super_block_start
BMFORCEINLINE RTYPE get_super_block_start(unsigned i) BMNOEXCEPT
Compute bit address of the first bit in a superblock.
Definition: bmfunc.h:158
bm::ptr_payload_t::i64
bm::id64_t i64
Definition: bmfunc.h:8848
bm::gap_bitset_sub_count
bm::id_t gap_bitset_sub_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount of bit block SUB masked by GAP block.
Definition: bmfunc.h:3641
bm::operation_functions::gapop_table_
static gap_operation_func_type gapop_table_[bm::set_END]
Definition: bmfunc.h:8355
bm::bit_for_each_4
void bit_for_each_4(T w, F &func)
Templated algorithm to unpacks octet based word into list of ON bit indexes.
Definition: bmfunc.h:295
bm::bit_block_rotate_left_1_unr
void bit_block_rotate_left_1_unr(bm::word_t *block) BMNOEXCEPT
Unrolled cyclic rotation of bit-block left by 1 bit.
Definition: bmfunc.h:4840
bm::bv_statistics::add_gap_block
void add_gap_block(unsigned capacity, unsigned length) BMNOEXCEPT
count gap block
Definition: bmfunc.h:76
bm::id_t
unsigned int id_t
Definition: bmconst.h:37
VECT_LOWER_BOUND_SCAN_U32
#define VECT_LOWER_BOUND_SCAN_U32(arr, target, from, to)
Definition: bmsse4.h:1797
bm::bitblock_sum_adapter
Bit-block sum adapter, takes values and sums it /internal.
Definition: bmfunc.h:8151
bm::bit_block_shift_r1
bool bit_block_shift_r1(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference)
Definition: bmfunc.h:4925
bm::bit_for_each
void bit_for_each(T w, F &func)
Templated algorithm to unpacks word into list of ON bit indexes.
Definition: bmfunc.h:365
VECT_INVERT_BLOCK
#define VECT_INVERT_BLOCK(first)
Definition: bmsse4.h:1734
bm::gap_is_all_zero
BMFORCEINLINE bool gap_is_all_zero(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-zero.
Definition: bmfunc.h:1072
bm::gap_operation_any_xor
BMFORCEINLINE unsigned gap_operation_any_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP XOR operation test.
Definition: bmfunc.h:5749
bm::gap_find_interval_end
bool gap_find_interval_end(const T *const BMRESTRICT buf, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the last 1 bit in the 111 interval of a GAP block.
Definition: bmfunc.h:1991
bm::set_XOR
Definition: bmconst.h:157
bm::all_set::all_set_block
Definition: bmfunc.h:844
VECT_OR_BLOCK_3WAY
#define VECT_OR_BLOCK_3WAY(dst, src1, src2)
Definition: bmsse4.h:1755
bm::bit_block_rotate_left_1
void bit_block_rotate_left_1(bm::word_t *block) BMNOEXCEPT
Definition: bmfunc.h:4824
bm::bitblock_store_adapter::bitblock_store_adapter
bitblock_store_adapter(bm::word_t *bit_block)
Definition: bmfunc.h:8140
bm::or_bit_block
void or_bit_block(unsigned *dest, unsigned bitpos, unsigned bitcount) BMNOEXCEPT
Sets bits to 1 in the bitblock.
Definition: bmfunc.h:3170
VECT_BITCOUNT_AND
#define VECT_BITCOUNT_AND(first, last, mask)
Definition: bmsse4.h:1722
bm::operation
operation
Bit operations.
Definition: bmconst.h:175
bmdef.h
Definitions(internal)
bm::bit_operation_sub_count
bm::id_t bit_operation_sub_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock SUB operation and calculates bitcount of the result.
Definition: bmfunc.h:6589
bm::bv_statistics::ptr_sub_blocks
size_t ptr_sub_blocks
Number of sub-blocks.
Definition: bmfunc.h:58
VECT_AND_DIGEST
#define VECT_AND_DIGEST(dst, src)
Definition: bmsse4.h:1740
bm::gap_operation_and
BMFORCEINLINE gap_word_t * gap_operation_and(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP AND operation.
Definition: bmfunc.h:5646
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2, unsigned bit_idx3) BMNOEXCEPT
Definition: bmfunc.h:411
bm::set_bitset
Simple bitset.
Definition: bmconst.h:203
bm::is_const_set_operation
bool is_const_set_operation(set_operation op) BMNOEXCEPT
Returns true if set operation is constant (bitcount)
Definition: bmfunc.h:817
bm::bit_COUNT_A::operator()
W operator()(W w1, W) BMNOEXCEPT
Definition: bmfunc.h:8315
bm::bit_block_shift_r1_unr
bool bit_block_shift_r1_unr(bm::word_t *BMRESTRICT block, bm::word_t *BMRESTRICT empty_acc, bm::word_t co_flag) BMNOEXCEPT
Right bit-shift of bit-block by 1 bit (loop unrolled)
Definition: bmfunc.h:4955
VECT_SHIFT_L1
#define VECT_SHIFT_L1(b, acc, co)
Definition: bmsse4.h:1800
bm::bit_operation_sub
bm::word_t * bit_operation_sub(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock SUB operation.
Definition: bmfunc.h:7182
bm::bit_find_last
unsigned bit_find_last(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT last) BMNOEXCEPT
BIT block find the last set bit (backward search)
Definition: bmfunc.h:7518
bm::gap_test_unr
unsigned gap_test_unr(const T *BMRESTRICT buf, const unsigned pos) BMNOEXCEPT
Tests if bit = pos is true. Analog of bm::gap_test with SIMD unrolling.
Definition: bmfunc.h:1298
VECT_XOR_BLOCK
#define VECT_XOR_BLOCK(dst, src)
Definition: bmsse4.h:1770
bm::block_find_interval_end
unsigned block_find_interval_end(const bm::word_t *BMRESTRICT block, unsigned nbit_from, unsigned *BMRESTRICT found_nbit) BMNOEXCEPT
Find end of the current 111 interval.
Definition: bmfunc.h:5428
bm::bit_to_gap
unsigned bit_to_gap(gap_word_t *BMRESTRICT dest, const unsigned *BMRESTRICT block, unsigned dest_len) BMNOEXCEPT
Convert bit block to GAP representation.
Definition: bmfunc.h:4248
VECT_IS_DIGEST_ZERO
#define VECT_IS_DIGEST_ZERO(start)
Definition: bmsse4.h:1791
bm::block_any_range
bool block_any_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if one bit is set in the block [left, right] Function check for block varieties.
Definition: bmfunc.h:5563
VECT_BLOCK_CHANGE
#define VECT_BLOCK_CHANGE(block, size)
Definition: bmsse4.h:1815
bm::or_op
BMFORCEINLINE unsigned or_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP or functor.
Definition: bmfunc.h:5617
bm::gap_max_bits
const unsigned gap_max_bits
Definition: bmconst.h:80
bm::improve_gap_levels
bool improve_gap_levels(const T *length, const T *length_end, T *glevel_len) BMNOEXCEPT
Finds optimal gap blocks lengths.
Definition: bmfunc.h:7932
bm::word_bitcount
BMFORCEINLINE bm::id_t word_bitcount(bm::id_t w) BMNOEXCEPT
Definition: bmfunc.h:197
bm::gap_add_to_bitset
void gap_add_to_bitset(unsigned *BMRESTRICT dest, const T *BMRESTRICT pcurr, unsigned len) BMNOEXCEPT
Adds(OR) GAP block to bitblock.
Definition: bmfunc.h:3436
bm::lower_bound_u64
unsigned lower_bound_u64(const unsigned long long *arr, unsigned long long target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned LONG array.
Definition: bmfunc.h:8780
bm::bit_block_xor_any
unsigned bit_block_xor_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition: bmfunc.h:6292
IS_VALID_ADDR
#define IS_VALID_ADDR(addr)
Definition: bmdef.h:152
bm::bit_find_first_if_1
bool bit_find_first_if_1(const bm::word_t *BMRESTRICT block, unsigned *BMRESTRICT first, bm::id64_t digest) BMNOEXCEPT
BIT block find the first set bit if only 1 bit is set.
Definition: bmfunc.h:7625
bm::lower_bound_u32
unsigned lower_bound_u32(const unsigned *arr, unsigned target, unsigned from, unsigned to) BMNOEXCEPT
Hybrid, binary-linear lower bound search in unsigned array.
Definition: bmfunc.h:8745
bm::gap_bitset_and_count
bm::id_t gap_bitset_and_count(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Compute bitcount of bit block AND masked by GAP block.
Definition: bmfunc.h:3583
bm::block_is_all_one_range
bool block_is_all_one_range(const bm::word_t *const BMRESTRICT block, unsigned left, unsigned right) BMNOEXCEPT
Returns "true" if all bits are 1 in the block [left, right] Function check for block varieties.
Definition: bmfunc.h:5303
VECT_SET_BLOCK
#define VECT_SET_BLOCK(dst, value)
Definition: bmsse4.h:1782
bm::bit_count_change
bm::id_t bit_count_change(bm::word_t w) BMNOEXCEPT
Definition: bmfunc.h:4517
bm::bit_SUB
Bit SUB functor.
Definition: bmfunc.h:8228
FULL_BLOCK_REAL_ADDR
#define FULL_BLOCK_REAL_ADDR
Definition: bmdef.h:148
bmutil.h
Bit manipulation primitives (internal)
VECT_BLOCK_SET_DIGEST
#define VECT_BLOCK_SET_DIGEST(dst, val)
Definition: bmsse4.h:1794
bm::bv_statistics::reset
void reset() BMNOEXCEPT
Reset statisctics.
Definition: bmfunc.h:96
bm::gap_set_array
unsigned gap_set_array(T *buf, const T *arr, unsigned len) BMNOEXCEPT
Convert array to GAP buffer.
Definition: bmfunc.h:3005
VECT_SET_BLOCK_BITS
#define VECT_SET_BLOCK_BITS(block, idx, start, stop)
Definition: bmsse4.h:1812
bm::bit_COUNT_SUB_BA::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8304
bm::bit_block_and_count
unsigned bit_block_and_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ANDs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6164
VECT_BIT_FIND_FIRST
#define VECT_BIT_FIND_FIRST(src, pos)
Definition: bmsse4.h:1826
bm::gap_init_range_block
void gap_init_range_block(T *buf, T from, T to, T value) BMNOEXCEPT
Init gap block so it has block in it (can be whole block)
Definition: bmfunc.h:3951
bm::bv_statistics::bv_count
size_t bv_count
Number of bit-vectors.
Definition: bmfunc.h:59
bm::bit_array_compute_gaps
unsigned bit_array_compute_gaps(const T *arr, unsigned len) BMNOEXCEPT
Compute number of GAPs in bit-array.
Definition: bmfunc.h:3066
bm::bit_COUNT_SUB_AB
Bit COUNT SUB AB functor.
Definition: bmfunc.h:8291
bm::gap_bfind
unsigned gap_bfind(const T *BMRESTRICT buf, unsigned pos, unsigned *BMRESTRICT is_set) BMNOEXCEPT
Definition: bmfunc.h:1220
bm::bit_COUNT_A
Bit COUNT A functor.
Definition: bmfunc.h:8313
bm::best_representation
bm::set_representation best_representation(unsigned bit_count, unsigned total_possible_bitcount, unsigned gap_count, unsigned block_size) BMNOEXCEPT
Choose best representation for a bit-block.
Definition: bmfunc.h:7785
bm::set_END
Definition: bmconst.h:168
bm::bit_block_find
unsigned bit_block_find(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the next 1 bit in the BIT block.
Definition: bmfunc.h:7467
bm::bit_operation_or_count
bm::id_t bit_operation_or_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock OR operation and calculates bitcount of the result.
Definition: bmfunc.h:6679
bm::pair::second
Second second
Definition: bmfunc.h:124
bm::bit_operation_and
bm::word_t * bit_operation_and(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
bitblock AND operation.
Definition: bmfunc.h:6476
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1, unsigned bit_idx2) BMNOEXCEPT
Definition: bmfunc.h:403
bm::dgap_2_gap
void dgap_2_gap(const T *BMRESTRICT dgap_buf, T *BMRESTRICT gap_buf, T gap_header=0) BMNOEXCEPT
Convert D-GAP buffer into GAP buffer.
Definition: bmfunc.h:2214
bm::bit_COUNT
Bit COUNT functor.
Definition: bmfunc.h:8246
bm::bit_scan_reverse32
unsigned bit_scan_reverse32(unsigned value) BMNOEXCEPT
Definition: bmutil.h:301
bm::set_block_shift
const unsigned set_block_shift
Definition: bmconst.h:55
bm::bit_operation_count_func_type
bm::id_t(* bit_operation_count_func_type)(const bm::word_t *BMRESTRICT, const bm::word_t *BMRESTRICT)
Definition: bmfunc.h:8345
bm::gap_limit
unsigned gap_limit(const T *BMRESTRICT buf, const T *BMRESTRICT glevel_len) BMNOEXCEPT
Returs GAP block capacity limit.
Definition: bmfunc.h:1130
bm::sse2_gap_test
unsigned sse2_gap_test(const unsigned short *BMRESTRICT buf, unsigned pos)
Hybrid binary search, starts as binary, then switches to scan.
Definition: bmsse2.h:461
bm::bit_block_or_any
unsigned bit_block_or_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function ORs two bitblocks and and tests for any bit. Function does not analyse availability of sourc...
Definition: bmfunc.h:6444
bm::d_copy_func::d_copy_func
d_copy_func(T *dg_buf)
Definition: bmfunc.h:2168
bm::bit_block_and_any
unsigned bit_block_and_any(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ANDs two bitblocks and tests for any bit. Function does not analyse availability of source b...
Definition: bmfunc.h:6213
bm::bit_block_find_interval_start
bool bit_block_find_interval_start(const bm::word_t *BMRESTRICT block, unsigned nbit, unsigned *BMRESTRICT pos) BMNOEXCEPT
Searches for the first 1 bit in the 111 interval of a BIT block.
Definition: bmfunc.h:5469
bm::bit_block_erase
void bit_block_erase(bm::word_t *block, unsigned bitpos, bool carry_over) BMNOEXCEPT
erase bit from position and shift the rest right with carryover
Definition: bmfunc.h:5033
bm::clear_bit
BMFORCEINLINE void clear_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition: bmfunc.h:3138
bm::count_leading_zeros_u64
unsigned count_leading_zeros_u64(bm::id64_t w) BMNOEXCEPT
64-bit bit-scan reverse
Definition: bmutil.h:353
bm::bit_COUNT_AND
Bit COUNT AND functor.
Definition: bmfunc.h:8257
bm::bitblock_get_adapter
Bit-block get adapter, takes bitblock and represents it as a get_32() accessor function.
Definition: bmfunc.h:8121
BMFORCEINLINE
#define BMFORCEINLINE
Definition: bmdef.h:203
bm
Definition: bm.h:76
bm::wordcmp0
int wordcmp0(T w1, T w2)
Lexicographical comparison of two words as bit strings (reference) Auxiliary implementation for testi...
Definition: bmfunc.h:998
VECT_COPY_BLOCK
#define VECT_COPY_BLOCK(dst, src)
Definition: bmsse4.h:1776
bm::gap_bitset_or_any
bm::id_t gap_bitset_or_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block OR masked by GAP block.
Definition: bmfunc.h:3822
IS_FULL_BLOCK
#define IS_FULL_BLOCK(addr)
Definition: bmdef.h:153
bm::bit_block_xor_count
unsigned bit_block_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function XORs two bitblocks and computes the bitcount. Function does not analyse availability of sour...
Definition: bmfunc.h:6243
bm::word_select64
unsigned word_select64(bm::id64_t w, unsigned rank) BMNOEXCEPT
word find index of the rank-th bit set by bit-testing
Definition: bmfunc.h:624
bm::copy_to_array_functor::operator()
void operator()(unsigned bit_idx0, unsigned bit_idx1) BMNOEXCEPT
Definition: bmfunc.h:396
bm::bit_block_shift_r1_and_unr
bool bit_block_shift_r1_and_unr(bm::word_t *BMRESTRICT block, bm::word_t co_flag, const bm::word_t *BMRESTRICT mask_block, bm::id64_t *BMRESTRICT digest) BMNOEXCEPT
Right bit-shift bitblock by 1 bit (reference) + AND.
Definition: bmfunc.h:5161
bm::gap_operation_xor
gap_word_t * gap_operation_xor(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP XOR operation.
Definition: bmfunc.h:5712
bm::bv_statistics::gap_levels
gap_word_t gap_levels[bm::gap_levels]
GAP block lengths in the bvect.
Definition: bmfunc.h:63
bm::bit_find_first_diff
bool bit_find_first_diff(const bm::word_t *BMRESTRICT blk1, const bm::word_t *BMRESTRICT blk2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two bit-blocks.
Definition: bmfunc.h:4106
bm::operation_functions::gap_op_to_bit
static gap_operation_to_bitset_func_type gap_op_to_bit(unsigned i)
Definition: bmfunc.h:8360
VECT_AND_DIGEST_2WAY
#define VECT_AND_DIGEST_2WAY(dst, src1, src2)
Definition: bmsse4.h:1746
bm::bit_block_or_count
unsigned bit_block_or_count(const bm::word_t *src1, const bm::word_t *src2) BMNOEXCEPT
Function ORs two bitblocks and computes the bitcount. Function does not analyse availability of sourc...
Definition: bmfunc.h:6396
bm::bit_operation_xor_count
bm::id_t bit_operation_xor_count(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock XOR operation and calculates bitcount of the result.
Definition: bmfunc.h:7349
bm::bitblock_store_adapter
Bit-block store adapter, takes bitblock and saves results into it.
Definition: bmfunc.h:8137
bm::bv_statistics::memory_used
size_t memory_used
memory usage for all blocks and service tables
Definition: bmfunc.h:61
bm::gap_count_sub
BMFORCEINLINE unsigned gap_count_sub(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2) BMNOEXCEPT
GAP bitcount SUB (AND NOT) operation test.
Definition: bmfunc.h:5877
bm::bit_scan_reverse
unsigned bit_scan_reverse(T value) BMNOEXCEPT
Definition: bmutil.h:415
bm::block_find_first_diff
bool block_find_first_diff(const bm::word_t *BMRESTRICT blk, const bm::word_t *BMRESTRICT arg_blk, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two blocks (GAP or bit)
Definition: bmfunc.h:8030
VECT_IS_ZERO_BLOCK
#define VECT_IS_ZERO_BLOCK(dst)
Definition: bmsse4.h:1785
bm::bitscan_wave
unsigned short bitscan_wave(const bm::word_t *BMRESTRICT w_ptr, unsigned char *BMRESTRICT bits) BMNOEXCEPT
Unpacks word wave (Nx 32-bit words)
Definition: bmfunc.h:8433
VECT_BITCOUNT_SUB
#define VECT_BITCOUNT_SUB(first, last, mask)
Definition: bmsse4.h:1731
bm::operation_functions
Definition: bmfunc.h:8350
bm::set_array0
array of 0 values
Definition: bmconst.h:206
bm::bit_AND::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8218
bm::bit_block_and_2way
bm::id64_t bit_block_and_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND
Definition: bmfunc.h:6101
bm::bit_operation_sub_any
bm::id_t bit_operation_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Performs bitblock test of SUB operation.
Definition: bmfunc.h:6644
bm::bit_block_xor
bm::id64_t bit_block_xor(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Plain bitblock XOR operation. Function does not analyse availability of source and destination blocks...
Definition: bmfunc.h:7240
bm::copy_to_array_functor
Adaptor to copy 1 bits to array.
Definition: bmfunc.h:386
bm::check_zero_digest
bool check_zero_digest(bm::id64_t digest, unsigned bitpos_from, unsigned bitpos_to) BMNOEXCEPT
check if all digest bits for the range [from..to] are 0
Definition: bmfunc.h:688
bm::set_word_mask
const unsigned set_word_mask
Definition: bmconst.h:72
bm::bit_block_copy
void bit_block_copy(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy operation.
Definition: bmfunc.h:5898
bm::bit_block_calc_count_to
bm::id_t bit_block_calc_count_to(const bm::word_t *block, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4767
bm::word_t
unsigned int word_t
Definition: bmconst.h:38
bm::ptr_payload_t::i16
unsigned short i16[4]
Definition: bmfunc.h:8849
BMRESTRICT
#define BMRESTRICT
Definition: bmdef.h:193
bm::gap_length
BMFORCEINLINE bm::gap_word_t gap_length(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Returs GAP block length.
Definition: bmfunc.h:1098
bm::bit_decode_cache::bit_decode_cache
bit_decode_cache()
Definition: bmfunc.h:138
VECT_IS_ONE_BLOCK
#define VECT_IS_ONE_BLOCK(dst)
Definition: bmsse4.h:1788
bm::bit_block_stream
void bit_block_stream(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src) BMNOEXCEPT
Bitblock copy/stream operation.
Definition: bmfunc.h:5917
bm::set_gap
GAP-RLE compression.
Definition: bmconst.h:204
bm::bit_block_calc_count_range
bm::id_t bit_block_calc_count_range(const bm::word_t *block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4698
bm::gap_is_all_one
BMFORCEINLINE bool gap_is_all_one(const bm::gap_word_t *BMRESTRICT buf) BMNOEXCEPT
Checks if GAP block is all-one.
Definition: bmfunc.h:1085
bm::bit_block_or_2way
bool bit_block_or_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
2 way (target := source1 | source2) bitblock OR operation.
Definition: bmfunc.h:6786
bm::bit_block_shift_l1
bool bit_block_shift_l1(bm::word_t *block, bm::word_t *empty_acc, bm::word_t co_flag) BMNOEXCEPT
Left bit-shift bitblock by 1 bit (reference)
Definition: bmfunc.h:4980
bm::bit_block_calc_change
unsigned bit_block_calc_change(const bm::word_t *block) BMNOEXCEPT
Definition: bmfunc.h:4607
bm::bit_block_sub_any
unsigned bit_block_sub_any(const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2) BMNOEXCEPT
Function SUBs two bitblocks and and tests for any bit. Function does not analyse availability of sour...
Definition: bmfunc.h:6367
bm::gap_shift_r1
bool gap_shift_r1(T *BMRESTRICT buf, unsigned co_flag, unsigned *BMRESTRICT new_len) BMNOEXCEPT
Right shift GAP block by 1 bit.
Definition: bmfunc.h:2897
bm::for_each_block
void for_each_block(T ***root, unsigned size1, F &f, BLOCK_IDX start)
Definition: bmfunc.h:1647
bm::bit_block_and_5way
bm::id64_t bit_block_and_5way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src0, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, const bm::word_t *BMRESTRICT src3, bm::id64_t digest) BMNOEXCEPT
digest based bit-block AND 5-way
Definition: bmfunc.h:6034
bm::bmi_bslr_u64
BMFORCEINLINE unsigned long long bmi_bslr_u64(unsigned long long w) BMNOEXCEPT
Definition: bmutil.h:332
bm::d_copy_func::dgap_buf_
T * dgap_buf_
Definition: bmfunc.h:2171
bm::bitscan_popcnt
unsigned short bitscan_popcnt(bm::id_t w, B *bits, unsigned short offs) BMNOEXCEPT
Unpacks word into list of ON bit indexes using popcnt method.
Definition: bmfunc.h:475
bm::bit_find_rank
SIZE_TYPE bit_find_rank(const bm::word_t *const block, SIZE_TYPE rank, unsigned nbit_from, unsigned &nbit_pos) BMNOEXCEPT
BIT block find position for the rank.
Definition: bmfunc.h:7682
bm::for_each_nzblock
void for_each_nzblock(T ***root, unsigned size1, F &f)
Definition: bmfunc.h:1382
bm::bv_statistics
Structure with statistical information about memory allocation footprint, serialization projection,...
Definition: bmfunc.h:54
bm::bit_COUNT_OR::operator()
W operator()(W w1, W w2) BMNOEXCEPT
Definition: bmfunc.h:8281
bm::gap_bit_count
unsigned gap_bit_count(const T *buf, unsigned dsize=0) BMNOEXCEPT
Calculates number of bits ON in GAP buffer.
Definition: bmfunc.h:1773
VECT_SUB_DIGEST
#define VECT_SUB_DIGEST(dst, src)
Definition: bmsse4.h:1764
bm::bit_block_sub_2way
bm::id64_t bit_block_sub_2way(bm::word_t *BMRESTRICT dst, const bm::word_t *BMRESTRICT src1, const bm::word_t *BMRESTRICT src2, bm::id64_t digest) BMNOEXCEPT
digest based bitblock SUB (AND NOT) operation (3 operand)
Definition: bmfunc.h:7115
bm::all_set::block_type
static bm::id64_t block_type(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:873
bm::bit_AND
Bit AND functor.
Definition: bmfunc.h:8216
bm::gap_bitset_and_any
bm::id_t gap_bitset_and_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT pcurr) BMNOEXCEPT
Bitcount test of bit block AND masked by GAP block.
Definition: bmfunc.h:3611
bm::bv_statistics::add
void add(const bv_statistics &st) BMNOEXCEPT
Sum data from another sttructure.
Definition: bmfunc.h:105
bm::gap_buff_op
void gap_buff_op(T *BMRESTRICT dest, const T *BMRESTRICT vect1, unsigned vect1_mask, const T *BMRESTRICT vect2, unsigned vect2_mask, F &f, unsigned &dlen) BMNOEXCEPT2
Abstract operation for GAP buffers. Receives functor F as a template argument.
Definition: bmfunc.h:2355
bm::set_array1
array of set 1 values
Definition: bmconst.h:205
bm::get_block_start
BMFORCEINLINE RTYPE get_block_start(unsigned i, unsigned j) BMNOEXCEPT
Compute bit address of the first bit in a block.
Definition: bmfunc.h:168
bm::gap_bitset_sub_any
bm::id_t gap_bitset_sub_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block SUB masked by GAP block.
Definition: bmfunc.h:3676
bm::bmfor_each
F bmfor_each(T first, T last, F f)
Definition: bmfunc.h:1682
bm::set_bit
BMFORCEINLINE void set_bit(unsigned *dest, unsigned bitpos) BMNOEXCEPT
Set 1 bit in a block.
Definition: bmfunc.h:3125
bm::bit_block_is_all_one_range
bool bit_block_is_all_one_range(const bm::word_t *const BMRESTRICT block, bm::word_t left, bm::word_t right) BMNOEXCEPT
Definition: bmfunc.h:4621
bm::and_op
BMFORCEINLINE unsigned and_op(unsigned v1, unsigned v2) BMNOEXCEPT2
GAP and functor.
Definition: bmfunc.h:5603
bm::sse42_test_all_zero_wave
BMFORCEINLINE bool sse42_test_all_zero_wave(const void *ptr)
check if wave of pointers is all NULL
Definition: bmsse4.h:595
bm::gap_find_first_diff
bool gap_find_first_diff(const T *BMRESTRICT buf1, const T *BMRESTRICT buf2, unsigned *BMRESTRICT pos) BMNOEXCEPT
Find first bit which is different between two GAP-blocks.
Definition: bmfunc.h:2314
bm::gap_levels
const unsigned gap_levels
Definition: bmconst.h:84
bm::gap_operation_or
gap_word_t * gap_operation_or(const gap_word_t *BMRESTRICT vect1, const gap_word_t *BMRESTRICT vect2, gap_word_t *BMRESTRICT tmp_buf, unsigned &dsize) BMNOEXCEPT
GAP OR operation.
Definition: bmfunc.h:5790
bm::bit_block_set
void bit_block_set(bm::word_t *BMRESTRICT dst, bm::word_t value) BMNOEXCEPT
Bitblock memset operation.
Definition: bmfunc.h:3841
bm::block_init_digest0
void block_init_digest0(bm::word_t *const block, bm::id64_t digest) BMNOEXCEPT
Init block with 000111000 pattren based on digest.
Definition: bmfunc.h:704
bm::gap_bitset_xor_any
bm::id_t gap_bitset_xor_any(const unsigned *BMRESTRICT block, const T *BMRESTRICT buf) BMNOEXCEPT
Compute bitcount test of bit block XOR masked by GAP block.
Definition: bmfunc.h:3752
bm::bitblock_sum_adapter::push_back
BMFORCEINLINE void push_back(bm::word_t w) BMNOEXCEPT
Definition: bmfunc.h:8156
bm::all_set::is_valid_block_addr
static BMFORCEINLINE bool is_valid_block_addr(const bm::word_t *bp) BMNOEXCEPT
Definition: bmfunc.h:898
bm::bit_COUNT_XOR
Bit COUNT XOR functor.
Definition: bmfunc.h:8268
VECT_OR_BLOCK_2WAY
#define VECT_OR_BLOCK_2WAY(dst, src1, src2)
Definition: bmsse4.h:1752
VECT_ARR_BLOCK_LOOKUP
#define VECT_ARR_BLOCK_LOOKUP(idx, size, nb, start)
Definition: bmsse4.h:1809
bm::gap_is_all_one_range
bool gap_is_all_one_range(const T *const BMRESTRICT buf, unsigned left, unsigned right) BMNOEXCEPT
Test if all bits are 1 in GAP buffer in the [left, right] range.
Definition: bmfunc.h:1910
bm::bit_block_calc_count
bm::id_t bit_block_calc_count(const bm::word_t *block, const bm::word_t *block_end) BMNOEXCEPT
Bitcount for bit string.
Definition: bmfunc.h:4482
bm::gap_max_buff_len
const unsigned gap_max_buff_len
Definition: bmconst.h:79
bm::operation_functions::bit_operation_count
static bit_operation_count_func_type bit_operation_count(unsigned i)
Definition: bmfunc.h:8372
bm::bitcount64_4way
unsigned bitcount64_4way(bm::id64_t x, bm::id64_t y, bm::id64_t u, bm::id64_t v) BMNOEXCEPT
Definition: bmfunc.h:254