pmt: initial 3.0.2 update
This commit is contained in:
629
include/libgnulib/ssfmalloc-bitmap.h
Executable file
629
include/libgnulib/ssfmalloc-bitmap.h
Executable file
@@ -0,0 +1,629 @@
|
||||
/* Simple and straight-forward malloc implementation (front end).
|
||||
|
||||
Copyright (C) 2020-2024 Free Software Foundation, Inc.
|
||||
|
||||
This file is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Lesser General Public License as
|
||||
published by the Free Software Foundation; either version 2.1 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This file is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>. */
|
||||
|
||||
/* Written by Bruno Haible <bruno@clisp.org>, 2020. */
|
||||
|
||||
/* ===================== Low-level functions for bitmaps ==================== */
|
||||
|
||||
/* A bitmap is represented as an array of uint32_t = 'unsigned int', each with
|
||||
32 bits. The bit i in the word with index j therefore represents bit
|
||||
k = 32 * j + i of the entire bit sequence. */
|
||||
|
||||
/* Initializes a bitmap. */
|
||||
static inline void
|
||||
init_bitmap_all_bits_clear (size_t num_words, uint32_t *words)
|
||||
{
|
||||
size_t i;
|
||||
for (i = 0; i < num_words; i++)
|
||||
words[i] = 0;
|
||||
}
|
||||
|
||||
/* Initializes a bitmap. */
|
||||
static inline void
|
||||
init_bitmap_all_bits_set (size_t num_words, uint32_t *words)
|
||||
{
|
||||
size_t i;
|
||||
for (i = 0; i < num_words; i++)
|
||||
words[i] = ~(uint32_t)0;
|
||||
}
|
||||
|
||||
/* Returns the smallest index k >= k0 for which the bit k is set in the bitmap
|
||||
consisting of num_words words. Returns (size_t)(-1) if there is none. */
|
||||
static size_t
|
||||
find_first_bit_set (size_t num_words, const uint32_t *words, size_t k0)
|
||||
{
|
||||
size_t j0 = k0 / 32;
|
||||
if (j0 < num_words)
|
||||
{
|
||||
size_t i0 = k0 % 32;
|
||||
const uint32_t *ptr = words + j0;
|
||||
/* Look at the word j0, ignoring the i0 least significant bits. */
|
||||
{
|
||||
size_t found = ffs (*ptr & (-1U << i0));
|
||||
if (found > 0)
|
||||
return 32 * j0 + (found - 1);
|
||||
}
|
||||
/* Look at the subsequent words. */
|
||||
const uint32_t *words_end = words + num_words;
|
||||
while (++ptr < words_end)
|
||||
{
|
||||
size_t found = ffs (*ptr);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - words) + (found - 1);
|
||||
}
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
|
||||
/* Returns the smallest index k >= 0 for which the bit packet of c consecutive
|
||||
bits (1 <= c <= 32) is all set in the bitmap consisting of num_words words.
|
||||
Returns (size_t)(-1) if there is none. */
|
||||
static size_t
|
||||
find_first_packet_set (size_t num_words, const uint32_t *words, size_t c)
|
||||
{
|
||||
const uint32_t *ptr = words;
|
||||
const uint32_t *words_end = words + num_words;
|
||||
switch (c)
|
||||
{
|
||||
case 1:
|
||||
{
|
||||
/* A simplified variant of find_first_bit_set. */
|
||||
for (; ptr < words_end; ptr++)
|
||||
{
|
||||
size_t found = ffs (*ptr);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 2:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t combined = longword & (longword >> 1);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 3:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t combined = longword & (longword >> 1) & (longword >> 2);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 4:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t combined = tmp1 & (tmp1 >> 2);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 5:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t combined = tmp2 & (longword >> 4);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 6:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t combined = tmp1 & (tmp1 >> 2) & (tmp1 >> 4);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 7:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t combined =
|
||||
tmp1 & (tmp1 >> 2) & (tmp1 >> 4) & (longword >> 6);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 8:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t combined = tmp2 & (tmp2 >> 4);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 9:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined = tmp3 & (longword >> 8);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 10:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined = tmp3 & (tmp1 >> 8);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 11:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined = tmp3 & (tmp1 >> 8) & (longword >> 10);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 12:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t combined = tmp2 & (tmp2 >> 4) & (tmp2 >> 8);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 13:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t combined =
|
||||
tmp2 & (tmp2 >> 4) & (tmp2 >> 8) & (longword >> 12);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 14:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t combined = tmp2 & (tmp2 >> 4) & (tmp2 >> 8) & (tmp1 >> 12);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 15:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
/* Optimized: Use 5, not 6, '&' operations. */
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (longword >> 4);
|
||||
uint64_t combined = tmp3 & (tmp3 >> 5) & (tmp3 >> 10);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 16:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined = tmp3 & (tmp3 >> 8);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 17:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (longword >> 16);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 18:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp1 >> 16);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 19:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp1 >> 16) & (longword >> 18);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 20:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp2 >> 16);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 21:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp2 >> 16) & (longword >> 20);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 22:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp2 >> 16) & (tmp1 >> 20);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 23:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined =
|
||||
tmp4 & (tmp2 >> 16) & (tmp1 >> 20) & (longword >> 22);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 24:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined = tmp3 & (tmp3 >> 8) & (tmp3 >> 16);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 25:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined =
|
||||
tmp3 & (tmp3 >> 8) & (tmp3 >> 16) & (longword >> 24);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 26:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined =
|
||||
tmp3 & (tmp3 >> 8) & (tmp3 >> 16) & (tmp1 >> 24);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 27:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
/* Optimized: Use 6, not 7, '&' operations. */
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (longword >> 8);
|
||||
uint64_t combined = tmp4 & (tmp4 >> 9) & (tmp4 >> 18);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 28:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined =
|
||||
tmp3 & (tmp3 >> 8) & (tmp3 >> 16) & (tmp2 >> 24);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 29:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t combined =
|
||||
tmp3 & (tmp3 >> 8) & (tmp3 >> 16) & (tmp2 >> 24) & (longword >> 28);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 30:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
/* Optimized: Use 6, not 7, '&' operations. */
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp1 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp4 >> 10) & (tmp4 >> 20);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 31:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined =
|
||||
tmp4 & (tmp3 >> 16) & (tmp2 >> 24) & (tmp1 >> 28) & (longword >> 30);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
case 32:
|
||||
{
|
||||
while (ptr < words_end)
|
||||
{
|
||||
uint64_t longword = *ptr++;
|
||||
if (likely (ptr < words_end))
|
||||
longword |= ((uint64_t) *ptr) << 32;
|
||||
uint64_t tmp1 = longword & (longword >> 1);
|
||||
uint64_t tmp2 = tmp1 & (tmp1 >> 2);
|
||||
uint64_t tmp3 = tmp2 & (tmp2 >> 4);
|
||||
uint64_t tmp4 = tmp3 & (tmp3 >> 8);
|
||||
uint64_t combined = tmp4 & (tmp4 >> 16);
|
||||
size_t found = ffsll (combined);
|
||||
if (found > 0)
|
||||
return 32 * (ptr - 1 - words) + (found - 1);
|
||||
}
|
||||
return (size_t)(-1);
|
||||
}
|
||||
default:
|
||||
/* Invalid argument. */
|
||||
abort ();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user