Skip to content
Snippets Groups Projects
Commit 57592ed7 authored by Thibault Allenet's avatar Thibault Allenet Committed by Olivier BICHLER
Browse files

Remove Compact data from utils - functions are integrated into WeightInterleaving Operator

parent e5fa197b
No related branches found
No related tags found
2 merge requests!318[Upd] release verision 0.5.0,!262Low bit support for ARM Cortex-M export
Hide whitespace changes
Inline Side-by-side
include/aidge/utils/CompactData.hpp deleted 100644 → 0
+ 0
− 105
View file @ e5fa197b
#ifndef __CONVERT_CUSTOM_DATA_H__
#define __CONVERT_CUSTOM_DATA_H__
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <cmath>
#include "aidge/utils/ErrorHandling.hpp"
namespace Aidge {
/**
* @brief Calculates the required size for the 8-bits`compactData` vector.
*
* This function determines the minimum number of bytes needed in `compactData`
* to store `dataSize` elements compacted to `nb_bits` bits each.
*
* @param dataSize The total number of elements in the input data array.
* @param nb_bits The number of bits to use for each compacted element (from 1 to 7).
* @return std::size_t The required size in bytes for `compactData`.
*/
std::size_t compactDataSize(std::size_t dataSize, std::uint8_t nb_bits) {
AIDGE_ASSERT(nb_bits > 0 && nb_bits < 8, "nb_bits must be between 1 and 4"); // Ensure valid bit width
// Calculate the number of `nb_bits` segments that can fit in an 8-bit byte.
const unsigned int nbSlot = 8 / nb_bits;
// Calculate the number of compacted bytes needed to store all data elements.
// The formula (dataSize + nbSlot - 1) / nbSlot effectively rounds up the division, ensuring that any remaining elements that don't fully fill a byte are accounted for.
std::size_t requiredSize = (dataSize + nbSlot - 1) / nbSlot;
return requiredSize;
}
/**
* @brief Compacts 8-bit data into a smaller bit-width representation.
*
* This function takes an array of 8-bit data and compacts it into smaller chunks
* based on the specified bit-width `nb_bits`. Each element in `compactData` will
* store multiple packed `nb_bits` segments extracted from `data`.
*
* @param data The input array of 8-bit values to be compacted.
* @param dataSize The size of the input `data` array.
* @param compactData The output array storing the compacted data.
* @param nb_bits The number of bits to extract from each `data` element (must be less than 8).
*/
void compact_data(const std::int8_t* data, std::size_t dataSize, std::int8_t* compactData, std::uint8_t nb_bits) {
AIDGE_ASSERT(nb_bits > 0 && nb_bits < 5, "Cannot compact with the given nb_bits"); // Ensure valid bit width
// Mask to extract `nb_bits` from each data element
const unsigned int mask = (1U << nb_bits) - 1;
// Calculate the number of `nb_bits` segments that fit into an 8-bit compacted value
const unsigned int nbSlot = 8 / nb_bits;
// Case nb_bits=3 or 4, then shift is 4
// Case nb_bits=2, then shift is 2
// Case nb_bits=1, then shift is 1
std::uint8_t shift = 8 / nbSlot;
const unsigned int nbFullCompactbytes = dataSize / nbSlot;
// Main loop to process data in groups of `nbSlot`
for (std::size_t i = 0; i < nbFullCompactbytes; ++i) {
std::int8_t compact = 0;
for (unsigned int j = 0; j < nbSlot; ++j) {
compact |= (data[i * nbSlot + j] & mask); // Apply mask to keep `nb_bits` only
// Shift only if not on the last slot to make room for the next `nb_bits`
if (j < nbSlot - 1) {
compact <<= shift;
}
}
// Store the compacted value in the output array
compactData[i] = compact;
}
// Handle any remaining data elements (if dataSize is not a multiple of nbSlot).
std::size_t remaining = dataSize % nbSlot;
if (remaining != 0) {
std::int8_t compact = 0;
for (std::size_t j = 0; j < remaining; ++j) {
compact |= (data[nbFullCompactbytes*nbSlot + j] & mask);
if (j < remaining - 1) {
compact <<= shift;
}
}
compact <<= (shift*(nbSlot - remaining));
// Store the last compacted value
compactData[dataSize / nbSlot] = compact;
}
}
}
#endif
\ No newline at end of file
python_binding/pybind_core.cpp
+ 0
− 2
View file @ 57592ed7
......@@ -17,7 +17,6 @@ namespace py = pybind11;
namespace Aidge {
void init_Random(py::module&);
void init_CompactData(py::module&);
void init_Data(py::module&);
void init_Database(py::module&);
void init_DataProvider(py::module&);
......@@ -102,7 +101,6 @@ void init_Filler(py::module&);
void init_Aidge(py::module& m) {
init_Random(m);
init_CompactData(m);
init_Data(m);
init_Database(m);
......
python_binding/utils/pybind_CompactData.cpp deleted 100644 → 0
+ 0
− 23
View file @ e5fa197b
/********************************************************************************
* Copyright (c) 2023 CEA-List
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*
********************************************************************************/
#include <pybind11/pybind11.h>
#include "aidge/utils/CompactData.hpp"
namespace py = pybind11;
namespace Aidge {
void init_CompactData(py::module &m) {
m.def("compact_data", &compact_data, py::arg("data"), py::arg("data_size"), py::arg("compact_data"), py::arg("nb_bits"),
"Compacts 8-bit data into smaller bit-width representation.");
}
} // namespace Aidge
unit_tests/utils/Test_CompactData.cpp deleted 100644 → 0
+ 0
− 137
View file @ e5fa197b
/********************************************************************************
* Copyright (c) 2023 CEA-List
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*
********************************************************************************/
#include <catch2/catch_test_macros.hpp>
#include <vector>
#include <cstdint>
#include <iostream>
#include "aidge/data/Tensor.hpp"
#include "aidge/utils/CompactData.hpp"
using namespace Aidge;
TEST_CASE("compactDataSize - Basic Tests", "[core][required_size]") {
SECTION("Single element cases") {
REQUIRE(Aidge::compactDataSize(1, 1) == 1); // 1 bit, needs 1 byte
REQUIRE(Aidge::compactDataSize(1, 7) == 1); // 7 bits, needs 1 byte
}
SECTION("Boundary cases for different nb_bits values") {
REQUIRE(Aidge::compactDataSize(8, 1) == 1); // 8 elements at 1 bit each, fits in 1 byte
REQUIRE(Aidge::compactDataSize(8, 2) == 2); // 8 elements at 2 bits each, needs 2 bytes
REQUIRE(Aidge::compactDataSize(8, 3) == 4); // 8 elements at 3 bits each, needs 4 bytes
REQUIRE(Aidge::compactDataSize(8, 4) == 4); // 8 elements at 4 bits each, needs 4 bytes
}
SECTION("Larger dataSize values") {
REQUIRE(Aidge::compactDataSize(16, 1) == 2); // 16 elements at 1 bit each, fits in 2 bytes
REQUIRE(Aidge::compactDataSize(16, 2) == 4); // 16 elements at 2 bits each, needs 4 bytes
REQUIRE(Aidge::compactDataSize(16, 3) == 8); // 16 elements at 3 bits each, needs 6 bytes
REQUIRE(Aidge::compactDataSize(16, 4) == 8); // 16 elements at 4 bits each, needs 8 bytes
}
SECTION("Odd dataSize values with varying nb_bits") {
REQUIRE(Aidge::compactDataSize(7, 1) == 1); // 7 elements at 1 bit each, fits in 1 byte
REQUIRE(Aidge::compactDataSize(7, 2) == 2); // 7 elements at 2 bits each, needs 2 bytes
REQUIRE(Aidge::compactDataSize(7, 3) == 4); // 7 elements at 3 bits each, needs 4 bytes
REQUIRE(Aidge::compactDataSize(7, 4) == 4); // 7 elements at 4 bits each, needs 4 bytes
}
SECTION("Minimum and maximum values for nb_bits") {
REQUIRE(Aidge::compactDataSize(5, 1) == 1); // 5 elements at 1 bit each, fits in 1 byte
}
SECTION("Edge Case - dataSize of 0 should result in 0 required size") {
REQUIRE(Aidge::compactDataSize(0, 1) == 0); // No data elements
}
}
TEST_CASE("Compact Data", "[core][compactdata]") {
SECTION("Basic Tests - 4-bit compaction") {
Tensor data = Array1D<std::int8_t, 4>{{static_cast<std::int8_t>(0x0F), static_cast<std::int8_t>(0xF5), static_cast<std::int8_t>(0xB3), static_cast<std::int8_t>(0x9C)}};
uint8_t nb_bits = 4;
std::size_t required_size = Aidge::compactDataSize(data.size(), nb_bits);
// std::size_t required_size = (data.size() + (8 / nb_bits) - 1) / (8 / nb_bits);
std::vector<int8_t> compactData(required_size);
Aidge::compact_data(static_cast<int8_t*>(data.getImpl()->rawPtr()), data.size(), static_cast<int8_t*>(compactData.data()), nb_bits);
// Expected result: each int8_t in compactData holds two 4-bit values.
std::vector<int8_t> expected = {{static_cast<int8_t>(0xF5), static_cast<int8_t>(0x3C)}};
REQUIRE(compactData == expected);
}
SECTION("Basic Tests - 2-bit compaction") {
Tensor data = Array1D<std::int8_t, 4>{{static_cast<std::int8_t>(0x03), static_cast<std::int8_t>(0x02), static_cast<std::int8_t>(0x01), static_cast<std::int8_t>(0x00)}};
uint8_t nb_bits = 2;
std::size_t required_size = Aidge::compactDataSize(data.size(), nb_bits);
std::vector<int8_t> compactData(required_size);
Aidge::compact_data(static_cast<int8_t*>(data.getImpl()->rawPtr()), data.size(), static_cast<int8_t*>(compactData.data()), nb_bits);
// Expected result: each int8_t in compactData holds four 2-bit values.
std::vector<int8_t> expected = {static_cast<int8_t>(0xE4)};
REQUIRE(compactData == expected);
}
SECTION("Edge Cases - Single element data") {
Tensor data = Array1D<std::int8_t, 1>{{static_cast<int8_t>(0x0F)}};
uint8_t nb_bits = 4;
std::size_t required_size = Aidge::compactDataSize(data.size(), nb_bits);
std::vector<int8_t> compactData(required_size);
Aidge::compact_data(static_cast<int8_t*>(data.getImpl()->rawPtr()), data.size(), static_cast<int8_t*>(compactData.data()), nb_bits);
// Expected result: should be a single 4-bit compacted value in one int8_t.
std::vector<int8_t> expected = {static_cast<int8_t>(0xF0)};
REQUIRE(compactData == expected);
}
SECTION("Edge Cases - Non-divisible dataSize for nbSlot") {
Tensor data = Array1D<std::int8_t, 3>{{static_cast<int8_t>(0x0F), static_cast<int8_t>(0xA5), static_cast<int8_t>(0x34)}};
uint8_t nb_bits = 4;
std::size_t required_size = Aidge::compactDataSize(data.size(), nb_bits);
std::vector<int8_t> compactData(required_size);
Aidge::compact_data(static_cast<int8_t*>(data.getImpl()->rawPtr()), data.size(), static_cast<int8_t*>(compactData.data()), nb_bits);
// Expected: last value padded (only three 4-bit values to fit in two int8_t).
std::vector<int8_t> expected = {static_cast<int8_t>(0xF5), static_cast<int8_t>(0x40)};
REQUIRE(compactData == expected);
}
SECTION("Edge Cases - Non-divisible dataSize for nbSlot") {
Tensor data = Array1D<std::int8_t, 3>{{static_cast<int8_t>(0x0F), static_cast<int8_t>(0x05), static_cast<int8_t>(0x04)}};
uint8_t nb_bits = 3;
std::size_t required_size = Aidge::compactDataSize(data.size(), nb_bits);
std::vector<int8_t> compactData(required_size);
Aidge::compact_data(static_cast<int8_t*>(data.getImpl()->rawPtr()), data.size(), static_cast<int8_t*>(compactData.data()), nb_bits);
// Expected: last value padded (only three 4-bit values to fit in two int8_t).
std::vector<int8_t> expected = {static_cast<int8_t>(0x75), static_cast<int8_t>(0x40)};
REQUIRE(compactData == expected);
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Copyright © Eclipse Foundation, Inc. All Rights Reserved.     Privacy Policy | Terms of Use | Copyright Agent