commit 02fa0219dd0cec8503cad03e71e144b466cc7c3f
parent 62f2c45b23ee191496e190973cf7ace7683be3af
Author: d.levin256@gmail.com <d.levin256@gmail.com>
Date: Mon, 2 Dec 2019 18:24:22 +0000
Move memory.hpp to cometa
Diffstat:
7 files changed, 300 insertions(+), 265 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -159,9 +159,11 @@ endif()
message(STATUS CPU_ARCH=${CPU_ARCH})
-add_executable(detect_cpu ${CMAKE_CURRENT_SOURCE_DIR}/cmake/detect_cpu.cpp)
-target_link_libraries(detect_cpu PRIVATE kfr)
-target_set_arch(detect_cpu PRIVATE generic)
+if (X86)
+ add_executable(detect_cpu ${CMAKE_CURRENT_SOURCE_DIR}/cmake/detect_cpu.cpp)
+ target_link_libraries(detect_cpu PRIVATE kfr)
+ target_set_arch(detect_cpu PRIVATE generic)
+endif ()
if (ENABLE_TESTS)
diff --git a/include/kfr/base/memory.hpp b/include/kfr/base/memory.hpp
@@ -25,259 +25,4 @@
*/
#pragma once
-#include "../simd/read_write.hpp"
-#include "../simd/types.hpp"
-#include <algorithm>
-#include <atomic>
-#include <memory>
-
-namespace kfr
-{
-
-namespace internal_generic
-{
-
-struct memory_statistics
-{
- std::atomic_uintptr_t allocation_count = ATOMIC_VAR_INIT(0);
- std::atomic_uintptr_t allocation_size = ATOMIC_VAR_INIT(0);
- std::atomic_uintptr_t deallocation_count = ATOMIC_VAR_INIT(0);
- std::atomic_uintptr_t deallocation_size = ATOMIC_VAR_INIT(0);
-};
-
-inline memory_statistics& get_memory_statistics()
-{
- static memory_statistics ms;
- return ms;
-}
-
-#pragma pack(push, 1)
-
-struct mem_header
-{
- u8 offset;
- u8 alignment;
- u8 reserved1;
- u8 reserved2;
- unsigned int references_uint;
- size_t size;
-
- KFR_MEM_INTRINSIC std::atomic_uint& references()
- {
- return reinterpret_cast<std::atomic_uint&>(references_uint);
- }
-}
-#ifdef CMT_GNU_ATTRIBUTES
-__attribute__((__packed__))
-#endif
-;
-
-#pragma pack(pop)
-
-inline mem_header* aligned_header(void* ptr) { return ptr_cast<mem_header>(ptr) - 1; }
-
-inline size_t aligned_size(void* ptr) { return aligned_header(ptr)->size; }
-
-inline void* aligned_malloc(size_t size, size_t alignment)
-{
- get_memory_statistics().allocation_count++;
- get_memory_statistics().allocation_size += size;
- void* ptr = malloc(size + (alignment - 1) + sizeof(mem_header));
- if (ptr == nullptr)
- return nullptr;
- void* aligned_ptr = advance(ptr, sizeof(mem_header));
- aligned_ptr = align_up(aligned_ptr, alignment);
- aligned_header(aligned_ptr)->alignment = static_cast<u8>(alignment > 255 ? 255 : alignment);
- aligned_header(aligned_ptr)->offset = static_cast<u8>(distance(aligned_ptr, ptr));
- aligned_header(aligned_ptr)->references() = 1;
- aligned_header(aligned_ptr)->size = size;
- return aligned_ptr;
-}
-
-inline void aligned_force_free(void* ptr)
-{
- get_memory_statistics().deallocation_count++;
- get_memory_statistics().deallocation_size += aligned_size(ptr);
- free(advance(ptr, -static_cast<ptrdiff_t>(aligned_header(ptr)->offset)));
-}
-
-inline void aligned_add_ref(void* ptr) { aligned_header(ptr)->references()++; }
-
-inline void aligned_free(void* ptr)
-{
- if (--aligned_header(ptr)->references() == 0)
- aligned_force_free(ptr);
-}
-
-inline void aligned_release(void* ptr) { aligned_free(ptr); }
-
-inline void* aligned_reallocate(void* ptr, size_t new_size, size_t alignment)
-{
- if (ptr)
- {
- if (new_size)
- {
- void* new_ptr = aligned_malloc(new_size, alignment);
- size_t old_size = aligned_size(ptr);
- memcpy(new_ptr, ptr, std::min(old_size, new_size));
- aligned_release(ptr);
- return new_ptr;
- }
- else
- {
- aligned_release(ptr);
- return nullptr;
- }
- }
- else
- {
- if (new_size)
- {
- return internal_generic::aligned_malloc(new_size, alignment);
- }
- else
- {
- return nullptr; // do nothing
- }
- }
-}
-} // namespace internal_generic
-
-/// @brief Allocates aligned memory
-template <typename T = void, size_t alignment = platform<>::native_cache_alignment>
-KFR_INTRINSIC T* aligned_allocate(size_t size = 1)
-{
- T* ptr = static_cast<T*>(CMT_ASSUME_ALIGNED(
- internal_generic::aligned_malloc(std::max(alignment, size * details::elementsize<T>()), alignment),
- alignment));
- return ptr;
-}
-
-/// @brief Deallocates aligned memory
-template <typename T = void>
-KFR_INTRINSIC void aligned_deallocate(T* ptr)
-{
- return internal_generic::aligned_free(ptr);
-}
-
-namespace internal_generic
-{
-template <typename T>
-struct aligned_deleter
-{
- KFR_MEM_INTRINSIC void operator()(T* ptr) const { aligned_deallocate(ptr); }
-};
-} // namespace internal_generic
-
-template <typename T>
-struct autofree
-{
- KFR_MEM_INTRINSIC autofree() {}
- explicit KFR_MEM_INTRINSIC autofree(size_t size) : ptr(aligned_allocate<T>(size)) {}
- autofree(const autofree&) = delete;
- autofree& operator=(const autofree&) = delete;
- autofree(autofree&&) CMT_NOEXCEPT = default;
- autofree& operator=(autofree&&) CMT_NOEXCEPT = default;
- KFR_MEM_INTRINSIC T& operator[](size_t index) CMT_NOEXCEPT { return ptr[index]; }
- KFR_MEM_INTRINSIC const T& operator[](size_t index) const CMT_NOEXCEPT { return ptr[index]; }
-
- template <typename U = T>
- KFR_MEM_INTRINSIC U* data() CMT_NOEXCEPT
- {
- return ptr_cast<U>(ptr.get());
- }
- template <typename U = T>
- KFR_MEM_INTRINSIC const U* data() const CMT_NOEXCEPT
- {
- return ptr_cast<U>(ptr.get());
- }
-
- std::unique_ptr<T[], internal_generic::aligned_deleter<T>> ptr;
-};
-
-#ifdef KFR_USE_STD_ALLOCATION
-
-template <typename T>
-using allocator = std::allocator<T>;
-
-#else
-
-/// @brief Aligned allocator
-template <typename T>
-struct allocator
-{
- using value_type = T;
- using pointer = T*;
- using const_pointer = const T*;
- using reference = T&;
- using const_reference = const T&;
- using size_type = std::size_t;
- using difference_type = std::ptrdiff_t;
-
- template <typename U>
- struct rebind
- {
- using other = allocator<U>;
- };
- constexpr allocator() CMT_NOEXCEPT = default;
- constexpr allocator(const allocator&) CMT_NOEXCEPT = default;
- template <typename U>
- constexpr allocator(const allocator<U>&) CMT_NOEXCEPT
- {
- }
- pointer allocate(size_type n) const
- {
- pointer result = aligned_allocate<value_type>(n);
- if (!result)
- CMT_THROW(std::bad_alloc());
- return result;
- }
- void deallocate(pointer p, size_type) { aligned_deallocate(p); }
-};
-
-template <typename T1, typename T2>
-constexpr inline bool operator==(const allocator<T1>&, const allocator<T2>&) CMT_NOEXCEPT
-{
- return true;
-}
-template <typename T1, typename T2>
-constexpr inline bool operator!=(const allocator<T1>&, const allocator<T2>&) CMT_NOEXCEPT
-{
- return false;
-}
-
-#endif
-
-struct aligned_new
-{
- inline static void* operator new(size_t size) noexcept { return aligned_allocate(size); }
- inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); }
-
-#ifdef __cpp_aligned_new
- inline static void* operator new(size_t size, std::align_val_t al) noexcept
- {
- return internal_generic::aligned_malloc(size, std::max(size_t(32), static_cast<size_t>(al)));
- }
- inline static void operator delete(void* ptr, std::align_val_t al) noexcept
- {
- return internal_generic::aligned_free(ptr);
- }
-#endif
-};
-
-#define KFR_CLASS_REFCOUNT(cl) \
- \
-public: \
- void addref() const { m_refcount++; } \
- void release() const \
- { \
- if (--m_refcount == 0) \
- { \
- delete this; \
- } \
- } \
- \
-private: \
- mutable std::atomic_uintptr_t m_refcount = ATOMIC_VAR_INIT(0);
-
-} // namespace kfr
+#include "../cometa/memory.hpp"
diff --git a/include/kfr/base/pointer.hpp b/include/kfr/base/pointer.hpp
@@ -129,7 +129,7 @@ public:
#ifdef __cpp_aligned_new
static void operator delete (void* p, std::align_val_t al) noexcept
{
- internal_generic::aligned_release(p);
+ details::aligned_release(p);
}
#endif
diff --git a/include/kfr/cometa.hpp b/include/kfr/cometa.hpp
@@ -2128,3 +2128,4 @@ CMT_PRAGMA_GNU(GCC diagnostic pop)
CMT_PRAGMA_GNU(GCC diagnostic pop)
CMT_PRAGMA_MSVC(warning(pop))
+
diff --git a/include/kfr/cometa/function.hpp b/include/kfr/cometa/function.hpp
@@ -4,6 +4,14 @@
#pragma once
#include "../cometa.hpp"
+#include "memory.hpp"
+#include <cstdlib>
+#include <cstddef>
+#include <type_traits>
+#include <memory>
+#if CMT_HAS_EXCEPTIONS
+#include <functional>
+#endif
namespace cometa
{
@@ -55,13 +63,16 @@ struct function_abstract
template <typename Fn, typename R, typename... Args>
struct function_impl : public function_abstract<R, Args...>
{
- inline static void* operator new(size_t size) noexcept { return std::aligned_alloc(size, alignof(Fn)); }
- inline static void operator delete(void* ptr) noexcept { return std::free(ptr); }
+ inline static void* operator new(size_t size) noexcept { return aligned_allocate(size, alignof(Fn)); }
+ inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); }
inline static void* operator new(size_t size, std::align_val_t al) noexcept
{
- return std::aligned_alloc(size, static_cast<size_t>(al));
+ return aligned_allocate(size, static_cast<size_t>(al));
+ }
+ inline static void operator delete(void* ptr, std::align_val_t al) noexcept
+ {
+ return aligned_deallocate(ptr);
}
- inline static void operator delete(void* ptr, std::align_val_t al) noexcept { return std::free(ptr); }
template <typename Fn_>
function_impl(Fn_ fn) : fn(std::forward<Fn_>(fn))
@@ -82,7 +93,7 @@ struct function<R(Args...)>
template <typename Fn, typename = std::enable_if_t<std::is_invocable_r_v<R, Fn, Args...> &&
!std::is_same_v<std::decay_t<Fn>, function>>>
- function(Fn fn) : impl(new internal::function_impl<std::decay_t<Fn>, R, Args...>(std::move(fn)))
+ function(Fn fn) : impl(new details::function_impl<std::decay_t<Fn>, R, Args...>(std::move(fn)))
{
}
@@ -96,11 +107,16 @@ struct function<R(Args...)>
R operator()(Args... args) const
{
+#if CMT_HAS_EXCEPTIONS
if (impl)
{
return impl->operator()(std::forward<Args>(args)...);
}
throw std::bad_function_call();
+#else
+ // With exceptions disabled let it crash. To prevent this, check first
+ return impl->operator()(std::forward<Args>(args)...);
+#endif
}
[[nodiscard]] explicit operator bool() const { return !!impl; }
diff --git a/include/kfr/cometa/memory.hpp b/include/kfr/cometa/memory.hpp
@@ -0,0 +1,269 @@
+/** @addtogroup cometa
+ * @{
+ */
+#pragma once
+
+#include "numeric.hpp"
+#include <algorithm>
+#include <atomic>
+#include <cstdint>
+#include <memory>
+
+namespace cometa
+{
+
+namespace details
+{
+
+struct memory_statistics
+{
+ std::atomic_uintptr_t allocation_count = ATOMIC_VAR_INIT(0);
+ std::atomic_uintptr_t allocation_size = ATOMIC_VAR_INIT(0);
+ std::atomic_uintptr_t deallocation_count = ATOMIC_VAR_INIT(0);
+ std::atomic_uintptr_t deallocation_size = ATOMIC_VAR_INIT(0);
+};
+
+inline memory_statistics& get_memory_statistics()
+{
+ static memory_statistics ms;
+ return ms;
+}
+
+#pragma pack(push, 1)
+
+struct mem_header
+{
+ u8 offset;
+ u8 alignment;
+ u8 reserved1;
+ u8 reserved2;
+ unsigned int references_uint;
+ size_t size;
+
+ CMT_MEM_INTRINSIC std::atomic_uint& references()
+ {
+ return reinterpret_cast<std::atomic_uint&>(references_uint);
+ }
+}
+#ifdef CMT_GNU_ATTRIBUTES
+__attribute__((__packed__))
+#endif
+;
+
+#pragma pack(pop)
+
+inline mem_header* aligned_header(void* ptr) { return ptr_cast<mem_header>(ptr) - 1; }
+
+inline size_t aligned_size(void* ptr) { return aligned_header(ptr)->size; }
+
+inline void* aligned_malloc(size_t size, size_t alignment)
+{
+ get_memory_statistics().allocation_count++;
+ get_memory_statistics().allocation_size += size;
+ void* ptr = malloc(size + (alignment - 1) + sizeof(mem_header));
+ if (ptr == nullptr)
+ return nullptr;
+ void* aligned_ptr = advance(ptr, sizeof(mem_header));
+ aligned_ptr = align_up(aligned_ptr, alignment);
+ aligned_header(aligned_ptr)->alignment = static_cast<u8>(alignment > 255 ? 255 : alignment);
+ aligned_header(aligned_ptr)->offset = static_cast<u8>(distance(aligned_ptr, ptr));
+ aligned_header(aligned_ptr)->references() = 1;
+ aligned_header(aligned_ptr)->size = size;
+ return aligned_ptr;
+}
+
+inline void aligned_force_free(void* ptr)
+{
+ get_memory_statistics().deallocation_count++;
+ get_memory_statistics().deallocation_size += aligned_size(ptr);
+ free(advance(ptr, -static_cast<ptrdiff_t>(aligned_header(ptr)->offset)));
+}
+
+inline void aligned_add_ref(void* ptr) { aligned_header(ptr)->references()++; }
+
+inline void aligned_free(void* ptr)
+{
+ if (--aligned_header(ptr)->references() == 0)
+ aligned_force_free(ptr);
+}
+
+inline void aligned_release(void* ptr) { aligned_free(ptr); }
+
+inline void* aligned_reallocate(void* ptr, size_t new_size, size_t alignment)
+{
+ if (ptr)
+ {
+ if (new_size)
+ {
+ void* new_ptr = aligned_malloc(new_size, alignment);
+ size_t old_size = aligned_size(ptr);
+ memcpy(new_ptr, ptr, std::min(old_size, new_size));
+ aligned_release(ptr);
+ return new_ptr;
+ }
+ else
+ {
+ aligned_release(ptr);
+ return nullptr;
+ }
+ }
+ else
+ {
+ if (new_size)
+ {
+ return details::aligned_malloc(new_size, alignment);
+ }
+ else
+ {
+ return nullptr; // do nothing
+ }
+ }
+}
+} // namespace details
+
+/// @brief Allocates aligned memory
+template <typename T = void, size_t alignment = 64>
+CMT_INTRINSIC T* aligned_allocate(size_t size = 1)
+{
+ T* ptr = static_cast<T*>(CMT_ASSUME_ALIGNED(
+ details::aligned_malloc(std::max(alignment, size * details::elementsize<T>()), alignment),
+ alignment));
+ return ptr;
+}
+/// @brief Allocates aligned memory
+template <typename T = void>
+CMT_INTRINSIC T* aligned_allocate(size_t size, size_t alignment)
+{
+ T* ptr = static_cast<T*>(CMT_ASSUME_ALIGNED(
+ details::aligned_malloc(std::max(alignment, size * details::elementsize<T>()), alignment),
+ alignment));
+ return ptr;
+}
+
+/// @brief Deallocates aligned memory
+template <typename T = void>
+CMT_INTRINSIC void aligned_deallocate(T* ptr)
+{
+ return details::aligned_free(ptr);
+}
+
+namespace details
+{
+template <typename T>
+struct aligned_deleter
+{
+ CMT_MEM_INTRINSIC void operator()(T* ptr) const { aligned_deallocate(ptr); }
+};
+} // namespace details
+
+template <typename T>
+struct autofree
+{
+ CMT_MEM_INTRINSIC autofree() {}
+ explicit CMT_MEM_INTRINSIC autofree(size_t size) : ptr(aligned_allocate<T>(size)) {}
+ autofree(const autofree&) = delete;
+ autofree& operator=(const autofree&) = delete;
+ autofree(autofree&&) CMT_NOEXCEPT = default;
+ autofree& operator=(autofree&&) CMT_NOEXCEPT = default;
+ CMT_MEM_INTRINSIC T& operator[](size_t index) CMT_NOEXCEPT { return ptr[index]; }
+ CMT_MEM_INTRINSIC const T& operator[](size_t index) const CMT_NOEXCEPT { return ptr[index]; }
+
+ template <typename U = T>
+ CMT_MEM_INTRINSIC U* data() CMT_NOEXCEPT
+ {
+ return ptr_cast<U>(ptr.get());
+ }
+ template <typename U = T>
+ CMT_MEM_INTRINSIC const U* data() const CMT_NOEXCEPT
+ {
+ return ptr_cast<U>(ptr.get());
+ }
+
+ std::unique_ptr<T[], details::aligned_deleter<T>> ptr;
+};
+
+#ifdef KFR_USE_STD_ALLOCATION
+
+template <typename T>
+using allocator = std::allocator<T>;
+
+#else
+
+/// @brief Aligned allocator
+template <typename T>
+struct allocator
+{
+ using value_type = T;
+ using pointer = T*;
+ using const_pointer = const T*;
+ using reference = T&;
+ using const_reference = const T&;
+ using size_type = std::size_t;
+ using difference_type = std::ptrdiff_t;
+
+ template <typename U>
+ struct rebind
+ {
+ using other = allocator<U>;
+ };
+ constexpr allocator() CMT_NOEXCEPT = default;
+ constexpr allocator(const allocator&) CMT_NOEXCEPT = default;
+ template <typename U>
+ constexpr allocator(const allocator<U>&) CMT_NOEXCEPT
+ {
+ }
+ pointer allocate(size_type n) const
+ {
+ pointer result = aligned_allocate<value_type>(n);
+ if (!result)
+ CMT_THROW(std::bad_alloc());
+ return result;
+ }
+ void deallocate(pointer p, size_type) { aligned_deallocate(p); }
+};
+
+template <typename T1, typename T2>
+constexpr inline bool operator==(const allocator<T1>&, const allocator<T2>&) CMT_NOEXCEPT
+{
+ return true;
+}
+template <typename T1, typename T2>
+constexpr inline bool operator!=(const allocator<T1>&, const allocator<T2>&) CMT_NOEXCEPT
+{
+ return false;
+}
+
+#endif
+
+struct aligned_new
+{
+ inline static void* operator new(size_t size) noexcept { return aligned_allocate(size); }
+ inline static void operator delete(void* ptr) noexcept { return aligned_deallocate(ptr); }
+
+#ifdef __cpp_aligned_new
+ inline static void* operator new(size_t size, std::align_val_t al) noexcept
+ {
+ return details::aligned_malloc(size, std::max(size_t(64), static_cast<size_t>(al)));
+ }
+ inline static void operator delete(void* ptr, std::align_val_t al) noexcept
+ {
+ return details::aligned_free(ptr);
+ }
+#endif
+};
+
+#define KFR_CLASS_REFCOUNT(cl) \
+ \
+public: \
+ void addref() const { m_refcount++; } \
+ void release() const \
+ { \
+ if (--m_refcount == 0) \
+ { \
+ delete this; \
+ } \
+ } \
+ \
+private: \
+ mutable std::atomic_uintptr_t m_refcount = ATOMIC_VAR_INIT(0);
+} // namespace cometa
+\ No newline at end of file
diff --git a/sources.cmake b/sources.cmake
@@ -36,6 +36,7 @@ set(
${PROJECT_SOURCE_DIR}/include/kfr/cometa/cstring.hpp
${PROJECT_SOURCE_DIR}/include/kfr/cometa/ctti.hpp
${PROJECT_SOURCE_DIR}/include/kfr/cometa/function.hpp
+ ${PROJECT_SOURCE_DIR}/include/kfr/cometa/memory.hpp
${PROJECT_SOURCE_DIR}/include/kfr/cometa/named_arg.hpp
${PROJECT_SOURCE_DIR}/include/kfr/cometa/numeric.hpp
${PROJECT_SOURCE_DIR}/include/kfr/cometa/range.hpp
