diff --git a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h index 139edc0902a1a..a855899f715d1 100644 --- a/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h +++ b/DataFormats/Detectors/TPC/include/DataFormatsTPC/ZeroSuppression.h @@ -33,7 +33,10 @@ struct TPCZSHDR { static constexpr unsigned int TPC_ZS_NBITS_V1 = 10; static constexpr unsigned int TPC_ZS_NBITS_V2 = 12; - unsigned char version; + unsigned char version; // ZS format version: + // 1: original row-based format with 10-bit ADC values + // 2: original row-based format with 12-bit ADC values + // 3: improved link-based format with extra META header unsigned char nTimeBins; unsigned short cruID; unsigned short timeOffset; diff --git a/Detectors/TPC/reconstruction/src/TPCTrackingDigitsPreCheck.cxx b/Detectors/TPC/reconstruction/src/TPCTrackingDigitsPreCheck.cxx index 376298417de23..734dc3e68a67a 100644 --- a/Detectors/TPC/reconstruction/src/TPCTrackingDigitsPreCheck.cxx +++ b/Detectors/TPC/reconstruction/src/TPCTrackingDigitsPreCheck.cxx @@ -61,6 +61,10 @@ TPCTrackingDigitsPreCheck::precheckModifiedData TPCTrackingDigitsPreCheck::runPr } for (int j = 0; j < d->nTPCDigits[i]; j++) { if (maxContTimeBin && d->tpcDigits[i][j].getTimeStamp() >= maxContTimeBin) { + static bool filterOutOfTF = getenv("TPC_WORKFLOW_FILTER_DIGITS_OUTSIDE_OF_TF") && atoi(getenv("TPC_WORKFLOW_FILTER_DIGITS_OUTSIDE_OF_TF")); + if (filterOutOfTF) { + continue; + } throw std::runtime_error("Digit time bin exceeds time frame length"); } if (updateDigits) { diff --git a/Detectors/TPC/workflow/src/ZSSpec.cxx b/Detectors/TPC/workflow/src/ZSSpec.cxx index fd04c29493980..274a4a51ddd98 100644 --- a/Detectors/TPC/workflow/src/ZSSpec.cxx +++ b/Detectors/TPC/workflow/src/ZSSpec.cxx @@ -102,7 +102,7 @@ DataProcessorSpec getZSEncoderSpec(std::vector const& tpcSectors, bool outR sizes.resize(NSectors * NEndpoints); const auto* dh = DataRefUtils::getHeader(pc.inputs().getFirstValid(true)); o2::InteractionRecord ir{0, dh->firstTForbit}; - o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputs->inputDigits, &zsoutput, sizes.data(), nullptr, &ir, _GPUParam, true, verify, config.configReconstruction.tpc.zsThreshold); + o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputs->inputDigits, &zsoutput, sizes.data(), nullptr, &ir, _GPUParam, true, verify, config.configReconstruction.tpc.zsThreshold); ZeroSuppressedContainer8kb* page = reinterpret_cast(zsoutput.get()); unsigned int offset = 0; for (unsigned int i = 0; i < NSectors; i++) { @@ -155,7 +155,7 @@ DataProcessorSpec getZSEncoderSpec(std::vector const& tpcSectors, bool outR writer.useCaching(); } ir = o2::raw::HBFUtils::Instance().getFirstSampledTFIR(); - o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputs->inputDigits, nullptr, nullptr, &writer, &ir, _GPUParam, true, false, config.configReconstruction.tpc.zsThreshold); + o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputs->inputDigits, nullptr, nullptr, &writer, &ir, _GPUParam, true, false, config.configReconstruction.tpc.zsThreshold); writer.writeConfFile("TPC", "RAWDATA", fmt::format("{}tpcraw.cfg", outDir)); } zsoutput.reset(nullptr); diff --git a/Detectors/TPC/workflow/src/convertDigitsToRawZS.cxx b/Detectors/TPC/workflow/src/convertDigitsToRawZS.cxx index 0ee919b6d2b71..15565394a6ce6 100644 --- a/Detectors/TPC/workflow/src/convertDigitsToRawZS.cxx +++ b/Detectors/TPC/workflow/src/convertDigitsToRawZS.cxx @@ -222,7 +222,7 @@ void convert(DigitArray& inputDigits, ProcessAttributes* processAttributes, o2:: o2::InteractionRecord ir = o2::raw::HBFUtils::Instance().getFirstSampledTFIR(); ir.bc = 0; // By convention the TF starts at BC = 0 - o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputDigits, nullptr, nullptr, &writer, &ir, mGPUParam, zs12bit, false, zsThreshold, processAttributes->padding); + o2::gpu::GPUReconstructionConvert::RunZSEncoder(inputDigits, nullptr, nullptr, &writer, &ir, mGPUParam, zs12bit, false, zsThreshold, processAttributes->padding); } int main(int argc, char** argv) diff --git a/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx b/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx index 74328baa6566c..c32d8bf0c3377 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx @@ -175,7 +175,75 @@ int GPUReconstructionConvert::GetMaxTimeBin(const GPUTrackingInOutZS& zspages) #endif } -void GPUReconstructionConvert::ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits) +// ------------------------------------------------- TPC ZS ------------------------------------------------- + +#ifdef GPUCA_TPC_GEOMETRY_O2 +namespace // anonymous +{ + +// ------------------------------------------------- TPC ZS General ------------------------------------------------- + +typedef std::array zsPage; + +struct zsEncoder { + int curRegion = 0; + unsigned int encodeBits = 0; + unsigned int zsVersion = 0; + unsigned int iSector; + o2::raw::RawFileWriter* raw; + const o2::InteractionRecord* ir; + const GPUParam& param; + bool padding; + int lastEndpoint = -2, lastTime = -1, lastRow = GPUCA_ROW_COUNT; + long hbf = -1, nexthbf = 0; + zsPage* page = nullptr; + unsigned char* pagePtr = nullptr; + TPCZSHDR* hdr = nullptr; + static void ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int orbit); +}; + +inline void zsEncoder::ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int orbit) +{ + o2::header::RAWDataHeader* rdh = (o2::header::RAWDataHeader*)ptr; + o2::raw::RDHUtils::setHeartBeatOrbit(*rdh, orbit); + o2::raw::RDHUtils::setHeartBeatBC(*rdh, shift); + o2::raw::RDHUtils::setMemorySize(*rdh, sizeof(o2::header::RAWDataHeader)); + o2::raw::RDHUtils::setVersion(*rdh, o2::raw::RDHUtils::getVersion()); + o2::raw::RDHUtils::setFEEID(*rdh, feeId); +} + +static inline auto ZSEncoderGetDigits(const GPUTrackingInOutDigits& in, int i) { return in.tpcDigits[i]; } +static inline auto ZSEncoderGetNDigits(const GPUTrackingInOutDigits& in, int i) { return in.nTPCDigits[i]; } +#ifdef GPUCA_O2_LIB +using DigitArray = std::array, o2::tpc::Sector::MAXSECTOR>; +static inline auto ZSEncoderGetDigits(const DigitArray& in, int i) { return in[i].data(); } +static inline auto ZSEncoderGetNDigits(const DigitArray& in, int i) { return in[i].size(); } +#endif // GPUCA_O2_LIB + +// ------------------------------------------------- TPC ZS Original Row-based ZS ------------------------------------------------- + +struct zsEncoderRow : public zsEncoder { + std::array streamBuffer = {}; + std::array streamBuffer8 = {}; + TPCZSTBHDR* curTBHdr = nullptr; + unsigned char* nSeq = nullptr; + int seqLen = 0; + int endpoint = 0, endpointStart = 0; + float encodeBitsFactor = 0; + int nRowsInTB = 0; + unsigned int streamSize = 0, streamSize8 = 0; + + static void ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits); + bool checkInput(std::vector& tmpBuffer, unsigned int k); + bool writeSubPage(); + void init(); + unsigned int encodeSequence(std::vector& tmpBuffer, unsigned int k); + + bool sort(const o2::tpc::Digit a, const o2::tpc::Digit b); + void decodePage(std::vector& outputBuffer, const zsPage* page, unsigned int endpoint, unsigned int firstOrbit); +}; + +inline void zsEncoderRow::ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits) { unsigned int byte = 0, bits = 0; unsigned int mask = (1 << nBits) - 1; @@ -194,33 +262,368 @@ void GPUReconstructionConvert::ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn = 0; } -#ifdef GPUCA_HAVE_O2HEADERS -void GPUReconstructionConvert::ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int orbit) +inline bool zsEncoderRow::sort(const o2::tpc::Digit a, const o2::tpc::Digit b) { - o2::header::RAWDataHeader* rdh = (o2::header::RAWDataHeader*)ptr; - o2::raw::RDHUtils::setHeartBeatOrbit(*rdh, orbit); - o2::raw::RDHUtils::setHeartBeatBC(*rdh, shift); - o2::raw::RDHUtils::setMemorySize(*rdh, sizeof(o2::header::RAWDataHeader)); - o2::raw::RDHUtils::setVersion(*rdh, o2::raw::RDHUtils::getVersion()); - o2::raw::RDHUtils::setFEEID(*rdh, feeId); + int endpointa = param.tpcGeometry.GetRegion(a.getRow()); + int endpointb = param.tpcGeometry.GetRegion(b.getRow()); + endpointa = 2 * endpointa + (a.getRow() >= param.tpcGeometry.GetRegionStart(endpointa) + param.tpcGeometry.GetRegionRows(endpointa) / 2); + endpointb = 2 * endpointb + (b.getRow() >= param.tpcGeometry.GetRegionStart(endpointb) + param.tpcGeometry.GetRegionRows(endpointb) / 2); + if (endpointa != endpointb) { + return endpointa <= endpointb; + } + if (a.getTimeStamp() != b.getTimeStamp()) { + return a.getTimeStamp() <= b.getTimeStamp(); + } + if (a.getRow() != b.getRow()) { + return a.getRow() <= b.getRow(); + } + return a.getPad() < b.getPad(); } -static inline auto ZSEncoderGetDigits(const GPUTrackingInOutDigits& in, int i) { return in.tpcDigits[i]; } -static inline auto ZSEncoderGetNDigits(const GPUTrackingInOutDigits& in, int i) { return in.nTPCDigits[i]; } -template void GPUReconstructionConvert::RunZSEncoder(const GPUTrackingInOutDigits&, std::unique_ptr*, unsigned int*, o2::raw::RawFileWriter*, const o2::InteractionRecord*, const GPUParam&, bool, bool, float, bool); +void zsEncoderRow::init() +{ + encodeBitsFactor = (1 << (encodeBits - 10)); +} + +bool zsEncoderRow::checkInput(std::vector& tmpBuffer, unsigned int k) +{ + seqLen = 1; + if (lastRow != tmpBuffer[k].getRow()) { + endpointStart = param.tpcGeometry.GetRegionStart(curRegion); + endpoint = curRegion * 2; + if (tmpBuffer[k].getRow() >= endpointStart + param.tpcGeometry.GetRegionRows(curRegion) / 2) { + endpoint++; + endpointStart += param.tpcGeometry.GetRegionRows(curRegion) / 2; + } + } + for (unsigned int l = k + 1; l < tmpBuffer.size(); l++) { + if (tmpBuffer[l].getRow() == tmpBuffer[k].getRow() && tmpBuffer[l].getTimeStamp() == tmpBuffer[k].getTimeStamp() && tmpBuffer[l].getPad() == tmpBuffer[l - 1].getPad() + 1) { + seqLen++; + } else { + break; + } + } + if (lastEndpoint >= 0 && lastTime != -1 && (int)hdr->nTimeBins + tmpBuffer[k].getTimeStamp() - lastTime >= 256) { + lastEndpoint = -1; + } + if (endpoint == lastEndpoint) { + unsigned int sizeChk = (unsigned int)(pagePtr - reinterpret_cast(page)); // already written + sizeChk += 2 * (nRowsInTB + (tmpBuffer[k].getRow() != lastRow && tmpBuffer[k].getTimeStamp() == lastTime)); // TB HDR + sizeChk += streamSize8; // in stream buffer + sizeChk += (lastTime != tmpBuffer[k].getTimeStamp()) && ((sizeChk + (streamSize * encodeBits + 7) / 8) & 1); // time bin alignment + sizeChk += (tmpBuffer[k].getTimeStamp() != lastTime || tmpBuffer[k].getRow() != lastRow) ? 3 : 0; // new row overhead + sizeChk += (lastTime != -1 && tmpBuffer[k].getTimeStamp() > lastTime) ? ((tmpBuffer[k].getTimeStamp() - lastTime - 1) * 2) : 0; // empty time bins + sizeChk += 2; // sequence metadata + const unsigned int streamSizeChkBits = streamSize * encodeBits + ((lastTime != tmpBuffer[k].getTimeStamp() && (streamSize * encodeBits) % 8) ? (8 - (streamSize * encodeBits) % 8) : 0); + if (sizeChk + (encodeBits + streamSizeChkBits + 7) / 8 > TPCZSHDR::TPC_ZS_PAGE_SIZE) { + lastEndpoint = -1; + } else if (sizeChk + (seqLen * encodeBits + streamSizeChkBits + 7) / 8 > TPCZSHDR::TPC_ZS_PAGE_SIZE) { + seqLen = ((TPCZSHDR::TPC_ZS_PAGE_SIZE - sizeChk) * 8 - streamSizeChkBits) / encodeBits; + } + // sizeChk += (seqLen * encodeBits + streamSizeChkBits + 7) / 8; + // printf("Endpoint %d (%d), Pos %d, Chk %d, Len %d, rows %d, StreamSize %d %d, time %d (%d), row %d (%d), pad %d\n", endpoint, lastEndpoint, (int) (pagePtr - reinterpret_cast(page)), sizeChk, seqLen, nRowsInTB, streamSize8, streamSize, (int)tmpBuffer[k].getTimeStamp(), lastTime, (int)tmpBuffer[k].getRow(), lastRow, tmpBuffer[k].getPad()); + } + return endpoint != lastEndpoint || tmpBuffer[k].getTimeStamp() != lastTime; +} + +bool zsEncoderRow::writeSubPage() +{ + if (pagePtr != reinterpret_cast(page)) { + pagePtr += 2 * nRowsInTB; + ZSstreamOut(streamBuffer.data(), streamSize, streamBuffer8.data(), streamSize8, encodeBits); + pagePtr = std::copy(streamBuffer8.data(), streamBuffer8.data() + streamSize8, pagePtr); + if (pagePtr - reinterpret_cast(page) > 8192) { + throw std::runtime_error("internal error during ZS encoding"); + } + streamSize8 = 0; + for (int l = 1; l < nRowsInTB; l++) { + curTBHdr->rowAddr1()[l - 1] += 2 * nRowsInTB; + } + } + return endpoint != lastEndpoint; +} + +unsigned int zsEncoderRow::encodeSequence(std::vector& tmpBuffer, unsigned int k) +{ + if (tmpBuffer[k].getTimeStamp() != lastTime) { + if (lastTime != -1) { + hdr->nTimeBins += tmpBuffer[k].getTimeStamp() - lastTime - 1; + pagePtr += (tmpBuffer[k].getTimeStamp() - lastTime - 1) * 2; + } + hdr->nTimeBins++; + if ((pagePtr - reinterpret_cast(page)) & 1) { + pagePtr++; + } + curTBHdr = reinterpret_cast(pagePtr); + curTBHdr->rowMask |= (endpoint & 1) << 15; + nRowsInTB = 0; + lastRow = GPUCA_ROW_COUNT; + } + if (tmpBuffer[k].getRow() != lastRow) { + curTBHdr->rowMask |= 1 << (tmpBuffer[k].getRow() - endpointStart); + ZSstreamOut(streamBuffer.data(), streamSize, streamBuffer8.data(), streamSize8, encodeBits); + if (nRowsInTB) { + curTBHdr->rowAddr1()[nRowsInTB - 1] = (pagePtr - reinterpret_cast(page)) + streamSize8; + } + nRowsInTB++; + nSeq = streamBuffer8.data() + streamSize8++; + *nSeq = 0; + } + (*nSeq)++; + streamBuffer8[streamSize8++] = tmpBuffer[k].getPad(); + streamBuffer8[streamSize8++] = streamSize + seqLen; + for (int l = 0; l < seqLen; l++) { + streamBuffer[streamSize++] = (unsigned short)(tmpBuffer[k + l].getChargeFloat() * encodeBitsFactor + 0.5f); + } + return seqLen; +} + +void zsEncoderRow::decodePage(std::vector& outputBuffer, const zsPage* decPage, unsigned int decEndpoint, unsigned int firstOrbit) +{ + const unsigned char* decPagePtr = reinterpret_cast(decPage); + const o2::header::RAWDataHeader* rdh = (const o2::header::RAWDataHeader*)decPagePtr; + if (o2::raw::RDHUtils::getMemorySize(*rdh) == sizeof(o2::header::RAWDataHeader)) { + return; + } + decPagePtr += sizeof(o2::header::RAWDataHeader); + const TPCZSHDR* decHDR = reinterpret_cast(decPagePtr); + decPagePtr += sizeof(*decHDR); + if (decHDR->version != 1 && decHDR->version != 2) { + throw std::runtime_error("invalid ZS version"); + } + const float decodeBitsFactor = 1.f / (1 << (encodeBits - 10)); + unsigned int mask = (1 << encodeBits) - 1; + int cruid = decHDR->cruID; + unsigned int sector = cruid / 10; + if (sector != iSector) { + throw std::runtime_error("invalid TPC sector"); + } + int region = cruid % 10; + if ((unsigned int)region != decEndpoint / 2) { + throw std::runtime_error("CRU ID / endpoint mismatch"); + } + int nRowsRegion = param.tpcGeometry.GetRegionRows(region); + + int timeBin = (decHDR->timeOffset + (unsigned long)(o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) - firstOrbit) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN; + for (int l = 0; l < decHDR->nTimeBins; l++) { + if ((decPagePtr - reinterpret_cast(decPage)) & 1) { + decPagePtr++; + } + const TPCZSTBHDR* tbHdr = reinterpret_cast(decPagePtr); + bool upperRows = tbHdr->rowMask & 0x8000; + if (tbHdr->rowMask != 0 && ((upperRows) ^ ((decEndpoint & 1) != 0))) { + throw std::runtime_error("invalid endpoint"); + } + const int rowOffset = param.tpcGeometry.GetRegionStart(region) + (upperRows ? (nRowsRegion / 2) : 0); + const int nRows = upperRows ? (nRowsRegion - nRowsRegion / 2) : (nRowsRegion / 2); + const int nRowsUsed = __builtin_popcount((unsigned int)(tbHdr->rowMask & 0x7FFF)); + decPagePtr += nRowsUsed ? (2 * nRowsUsed) : 2; + int rowPos = 0; + for (int m = 0; m < nRows; m++) { + if ((tbHdr->rowMask & (1 << m)) == 0) { + continue; + } + const unsigned char* rowData = rowPos == 0 ? decPagePtr : (reinterpret_cast(decPage) + tbHdr->rowAddr1()[rowPos - 1]); + const int nSeqRead = *rowData; + const unsigned char* adcData = rowData + 2 * nSeqRead + 1; + int nADC = (rowData[2 * nSeqRead] * encodeBits + 7) / 8; + decPagePtr += 1 + 2 * nSeqRead + nADC; + unsigned int byte = 0, bits = 0, pos10 = 0; + std::array decBuffer; + for (int n = 0; n < nADC; n++) { + byte |= *(adcData++) << bits; + bits += 8; + while (bits >= encodeBits) { + decBuffer[pos10++] = byte & mask; + byte = byte >> encodeBits; + bits -= encodeBits; + } + } + pos10 = 0; + for (int n = 0; n < nSeqRead; n++) { + const int decSeqLen = rowData[(n + 1) * 2] - (n ? rowData[n * 2] : 0); + for (int o = 0; o < decSeqLen; o++) { + outputBuffer.emplace_back(o2::tpc::Digit{0, (float)decBuffer[pos10++] * decodeBitsFactor, (tpccf::Row)(rowOffset + m), (tpccf::Pad)(rowData[n * 2 + 1] + o), timeBin + l}); + } + } + rowPos++; + } + } +} + +// ------------------------------------------------- TPC ZS Main Encoder ------------------------------------------------- + +template +struct zsEncoderRun : public T { + unsigned int run(std::vector* buffer, std::vector& tmpBuffer); + size_t compare(std::vector* buffer, std::vector& tmpBuffer); + + using T::checkInput; + using T::curRegion; + using T::decodePage; + using T::encodeBits; + using T::encodeSequence; + using T::endpoint; + using T::hbf; + using T::hdr; + using T::init; + using T::ir; + using T::iSector; + using T::lastEndpoint; + using T::lastRow; + using T::lastTime; + using T::nexthbf; + using T::padding; + using T::page; + using T::pagePtr; + using T::param; + using T::raw; + using T::sort; + using T::writeSubPage; + using T::ZSfillEmpty; + using T::zsVersion; +}; + +template +inline unsigned int zsEncoderRun::run(std::vector* buffer, std::vector& tmpBuffer) +{ + unsigned int totalPages = 0; + zsPage singleBuffer; #ifdef GPUCA_O2_LIB -using DigitArray = std::array, o2::tpc::Sector::MAXSECTOR>; -template void GPUReconstructionConvert::RunZSEncoder(const DigitArray&, std::unique_ptr*, unsigned int*, o2::raw::RawFileWriter*, const o2::InteractionRecord*, const GPUParam&, bool, bool, float, bool); -static inline auto ZSEncoderGetDigits(const DigitArray& in, int i) { return in[i].data(); } -static inline auto ZSEncoderGetNDigits(const DigitArray& in, int i) { return in[i].size(); } + int rawlnk = rdh_utils::UserLogicLinkID; +#else + int rawlnk = 15; #endif -static inline auto ZSEncoderGetTime(const o2::tpc::Digit a) { return a.getTimeStamp(); } -static inline auto ZSEncoderGetPad(const o2::tpc::Digit a) { return a.getPad(); } -static inline auto ZSEncoderGetRow(const o2::tpc::Digit a) { return a.getRow(); } -static inline auto ZSEncoderGetCharge(const o2::tpc::Digit a) { return a.getChargeFloat(); } + int bcShiftInFirstHBF = ir ? ir->bc : 0; + int orbitShift = ir ? ir->orbit : 0; + int rawcru = 0; + int rawendpoint = 0; + (void)(rawcru + rawendpoint); // avoid compiler warning + init(); + + std::sort(tmpBuffer.begin(), tmpBuffer.end(), [this](const o2::tpc::Digit a, const o2::tpc::Digit b) { return sort(a, b); }); + for (unsigned int k = 0; k <= tmpBuffer.size(); k++) { + bool mustWritePage = false, mustWriteSubPage = false; + if (k < tmpBuffer.size()) { + if (tmpBuffer[k].getTimeStamp() != lastTime) { + nexthbf = ((long)tmpBuffer[k].getTimeStamp() * LHCBCPERTIMEBIN + bcShiftInFirstHBF) / o2::constants::lhc::LHCMaxBunches; + if (nexthbf < 0) { + throw std::runtime_error("Received digit before the defined first orbit"); + } + if (hbf != nexthbf) { + lastEndpoint = -2; + mustWritePage = true; + } + } + if (lastRow != tmpBuffer[k].getRow()) { + curRegion = param.tpcGeometry.GetRegion(tmpBuffer[k].getRow()); + } + mustWriteSubPage = checkInput(tmpBuffer, k); + } else { + nexthbf = -1; + mustWritePage = true; + } + if (mustWritePage || mustWriteSubPage) { + mustWritePage |= writeSubPage(); + + if (page && mustWritePage) { + const rdh_utils::FEEIDType rawfeeid = rdh_utils::getFEEID(rawcru, rawendpoint, rawlnk); + size_t size = (padding || lastEndpoint == -1 || hbf == nexthbf) ? TPCZSHDR::TPC_ZS_PAGE_SIZE : (pagePtr - (unsigned char*)page); + size = CAMath::nextMultipleOf(size); +#ifdef GPUCA_O2_LIB + if (raw) { + raw->addData(rawfeeid, rawcru, rawlnk, rawendpoint, *ir + hbf * o2::constants::lhc::LHCMaxBunches, gsl::span((char*)page + sizeof(o2::header::RAWDataHeader), (char*)page + size), true); + } else #endif + { + o2::header::RAWDataHeader* rdh = (o2::header::RAWDataHeader*)page; + o2::raw::RDHUtils::setHeartBeatOrbit(*rdh, hbf + orbitShift); + o2::raw::RDHUtils::setHeartBeatBC(*rdh, bcShiftInFirstHBF); + o2::raw::RDHUtils::setMemorySize(*rdh, size); + o2::raw::RDHUtils::setVersion(*rdh, o2::raw::RDHUtils::getVersion()); + o2::raw::RDHUtils::setFEEID(*rdh, rawfeeid); + } + } + if (k >= tmpBuffer.size()) { + break; + } + } + if (mustWritePage) { + if (raw) { + page = &singleBuffer; + } else { + if (buffer[endpoint].size() == 0 && nexthbf > orbitShift) { + buffer[endpoint].emplace_back(); + ZSfillEmpty(&buffer[endpoint].back(), bcShiftInFirstHBF, rdh_utils::getFEEID(iSector * 10 + endpoint / 2, endpoint & 1, rawlnk), orbitShift); // Emplace empty page with RDH containing beginning of TF + totalPages++; + } + buffer[endpoint].emplace_back(); + page = &buffer[endpoint].back(); + } + hbf = nexthbf; + pagePtr = reinterpret_cast(page); + std::fill(page->begin(), page->end(), 0); + pagePtr += sizeof(o2::header::RAWDataHeader); + hdr = reinterpret_cast(pagePtr); + pagePtr += sizeof(*hdr); + hdr->version = zsVersion; + hdr->cruID = iSector * 10 + curRegion; + rawcru = iSector * 10 + curRegion; + rawendpoint = endpoint & 1; + hdr->timeOffset = (long)tmpBuffer[k].getTimeStamp() * LHCBCPERTIMEBIN - (long)hbf * o2::constants::lhc::LHCMaxBunches; + lastTime = -1; + lastEndpoint = endpoint; + totalPages++; + } + unsigned int nEncoded = encodeSequence(tmpBuffer, k); + lastTime = tmpBuffer[k].getTimeStamp(); + lastRow = tmpBuffer[k].getRow(); + hdr->nADCsamples += nEncoded; + k += nEncoded - 1; + } + if (!raw) { + for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { + if (buffer[j].size() == 0) { + buffer[j].emplace_back(); + ZSfillEmpty(&buffer[j].back(), bcShiftInFirstHBF, rdh_utils::getFEEID(iSector * 10 + j / 2, j & 1, rawlnk), orbitShift); + totalPages++; + } + } + } + return totalPages; +} + +template +size_t zsEncoderRun::compare(std::vector* buffer, std::vector& tmpBuffer) +{ + size_t nErrors = 0; + std::vector compareBuffer; + compareBuffer.reserve(tmpBuffer.size()); + for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { + unsigned int firstOrbit = o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)buffer[j].data()); + for (unsigned int k = 0; k < buffer[j].size(); k++) { + zsPage* decPage = &buffer[j][k]; + decodePage(compareBuffer, decPage, j, firstOrbit); + } + } + for (unsigned int j = 0; j < tmpBuffer.size(); j++) { + const float decodeBitsFactor = (1 << (encodeBits - 10)); + const float c = (float)((int)(tmpBuffer[j].getChargeFloat() * decodeBitsFactor + 0.5f)) / decodeBitsFactor; + int ok = c == compareBuffer[j].getChargeFloat() && (int)tmpBuffer[j].getTimeStamp() == (int)compareBuffer[j].getTimeStamp() && (int)tmpBuffer[j].getPad() == (int)compareBuffer[j].getPad() && (int)tmpBuffer[j].getRow() == (int)compareBuffer[j].getRow(); + if (ok) { + continue; + } + nErrors++; + printf("%4u: OK %d: Charge %3d %3d Time %4d %4d Pad %3d %3d Row %3d %3d\n", j, ok, + (int)c, (int)compareBuffer[j].getChargeFloat(), (int)tmpBuffer[j].getTimeStamp(), (int)compareBuffer[j].getTimeStamp(), (int)tmpBuffer[j].getPad(), (int)compareBuffer[j].getPad(), (int)tmpBuffer[j].getRow(), (int)compareBuffer[j].getRow()); + } + return nErrors; +} -template +} // anonymous namespace +#endif // GPUCA_TPC_GEOMETRY_O2 + +template void GPUReconstructionConvert::RunZSEncoder(const S& in, std::unique_ptr* outBuffer, unsigned int* outSizes, o2::raw::RawFileWriter* raw, const o2::InteractionRecord* ir, const GPUParam& param, bool zs12bit, bool verify, float threshold, bool padding) { // Pass in either outBuffer / outSizes, to fill standalone output buffers, or raw to use RawFileWriter @@ -229,315 +632,30 @@ void GPUReconstructionConvert::RunZSEncoder(const S& in, std::unique_ptr> buffer[NSLICES][GPUTrackingInOutZS::NENDPOINTS]; + std::vector buffer[NSLICES][GPUTrackingInOutZS::NENDPOINTS]; unsigned int totalPages = 0; size_t nErrors = 0; - int encodeBits = zs12bit ? TPCZSHDR::TPC_ZS_NBITS_V2 : TPCZSHDR::TPC_ZS_NBITS_V1; - const float encodeBitsFactor = (1 << (encodeBits - 10)); // clang-format off GPUCA_OPENMP(parallel for reduction(+ : totalPages) reduction(+ : nErrors)) // clang-format on for (unsigned int i = 0; i < NSLICES; i++) { - std::array singleBuffer; -#ifdef GPUCA_O2_LIB - int rawlnk = rdh_utils::UserLogicLinkID; -#else - int rawlnk = 15; -#endif - int bcShiftInFirstHBF = ir ? ir->bc : 0; - int orbitShift = ir ? ir->orbit : 0; - int rawcru = 0; - int rawendpoint = 0; - (void)(rawcru + rawendpoint); // avoid compiler warning - - std::vector tmpBuffer; - std::array streamBuffer; - std::array streamBuffer8; + std::vector tmpBuffer; tmpBuffer.resize(ZSEncoderGetNDigits(in, i)); if (threshold > 0.f) { - auto it = std::copy_if(ZSEncoderGetDigits(in, i), ZSEncoderGetDigits(in, i) + ZSEncoderGetNDigits(in, i), tmpBuffer.begin(), [threshold](auto& v) { return ZSEncoderGetCharge(v) >= threshold; }); + auto it = std::copy_if(ZSEncoderGetDigits(in, i), ZSEncoderGetDigits(in, i) + ZSEncoderGetNDigits(in, i), tmpBuffer.begin(), [threshold](auto& v) { return v.getChargeFloat() >= threshold; }); tmpBuffer.resize(std::distance(tmpBuffer.begin(), it)); } else { std::copy(ZSEncoderGetDigits(in, i), ZSEncoderGetDigits(in, i) + ZSEncoderGetNDigits(in, i), tmpBuffer.begin()); } - std::sort(tmpBuffer.begin(), tmpBuffer.end(), [¶m](const T a, const T b) { - int endpointa = param.tpcGeometry.GetRegion(ZSEncoderGetRow(a)); - int endpointb = param.tpcGeometry.GetRegion(ZSEncoderGetRow(b)); - endpointa = 2 * endpointa + (ZSEncoderGetRow(a) >= param.tpcGeometry.GetRegionStart(endpointa) + param.tpcGeometry.GetRegionRows(endpointa) / 2); - endpointb = 2 * endpointb + (ZSEncoderGetRow(b) >= param.tpcGeometry.GetRegionStart(endpointb) + param.tpcGeometry.GetRegionRows(endpointb) / 2); - if (endpointa != endpointb) { - return endpointa <= endpointb; - } - if (ZSEncoderGetTime(a) != ZSEncoderGetTime(b)) { - return ZSEncoderGetTime(a) <= ZSEncoderGetTime(b); - } - if (ZSEncoderGetRow(a) != ZSEncoderGetRow(b)) { - return ZSEncoderGetRow(a) <= ZSEncoderGetRow(b); - } - return ZSEncoderGetPad(a) < ZSEncoderGetPad(b); - }); - int lastEndpoint = -2, lastRow = GPUCA_ROW_COUNT, lastTime = -1; - long hbf = -1, nexthbf = 0; - std::array* page = nullptr; - TPCZSHDR* hdr = nullptr; - TPCZSTBHDR* tbHdr = nullptr; - unsigned char* pagePtr = nullptr; - unsigned char* nSeq = nullptr; - int nRowsInTB = 0; - int region = 0, endpoint = 0, endpointStart = 0; - unsigned int streamSize = 0, streamSize8 = 0; - for (unsigned int k = 0; k <= tmpBuffer.size(); k++) { - int seqLen = 1; - if (k < tmpBuffer.size()) { - if (lastRow != ZSEncoderGetRow(tmpBuffer[k])) { - region = param.tpcGeometry.GetRegion(ZSEncoderGetRow(tmpBuffer[k])); - endpointStart = param.tpcGeometry.GetRegionStart(region); - endpoint = region * 2; - if (ZSEncoderGetRow(tmpBuffer[k]) >= endpointStart + param.tpcGeometry.GetRegionRows(region) / 2) { - endpoint++; - endpointStart += param.tpcGeometry.GetRegionRows(region) / 2; - } - } - for (unsigned int l = k + 1; l < tmpBuffer.size(); l++) { - if (ZSEncoderGetRow(tmpBuffer[l]) == ZSEncoderGetRow(tmpBuffer[k]) && ZSEncoderGetTime(tmpBuffer[l]) == ZSEncoderGetTime(tmpBuffer[k]) && ZSEncoderGetPad(tmpBuffer[l]) == ZSEncoderGetPad(tmpBuffer[l - 1]) + 1) { - seqLen++; - } else { - break; - } - } - if (lastTime != -1 && (int)hdr->nTimeBins + ZSEncoderGetTime(tmpBuffer[k]) - lastTime >= 256) { - lastEndpoint = -1; - } - if (ZSEncoderGetTime(tmpBuffer[k]) != lastTime) { - nexthbf = ((long)ZSEncoderGetTime(tmpBuffer[k]) * LHCBCPERTIMEBIN + bcShiftInFirstHBF) / o2::constants::lhc::LHCMaxBunches; - if (nexthbf < 0) { - throw std::runtime_error("Received digit before the defined first orbit"); - } - if (hbf != nexthbf) { - lastEndpoint = -2; - } - } - if (endpoint == lastEndpoint) { - unsigned int sizeChk = (unsigned int)(pagePtr - reinterpret_cast(page)); // already written - sizeChk += 2 * (nRowsInTB + (ZSEncoderGetRow(tmpBuffer[k]) != lastRow && ZSEncoderGetTime(tmpBuffer[k]) == lastTime)); // TB HDR - sizeChk += streamSize8; // in stream buffer - sizeChk += (lastTime != ZSEncoderGetTime(tmpBuffer[k])) && ((sizeChk + (streamSize * encodeBits + 7) / 8) & 1); // time bin alignment - sizeChk += (ZSEncoderGetTime(tmpBuffer[k]) != lastTime || ZSEncoderGetRow(tmpBuffer[k]) != lastRow) ? 3 : 0; // new row overhead - sizeChk += (lastTime != -1 && ZSEncoderGetTime(tmpBuffer[k]) > lastTime) ? ((ZSEncoderGetTime(tmpBuffer[k]) - lastTime - 1) * 2) : 0; // empty time bins - sizeChk += 2; // sequence metadata - const unsigned int streamSizeChkBits = streamSize * encodeBits + ((lastTime != ZSEncoderGetTime(tmpBuffer[k]) && (streamSize * encodeBits) % 8) ? (8 - (streamSize * encodeBits) % 8) : 0); - if (sizeChk + (encodeBits + streamSizeChkBits + 7) / 8 > TPCZSHDR::TPC_ZS_PAGE_SIZE) { - lastEndpoint = -1; - } else if (sizeChk + (seqLen * encodeBits + streamSizeChkBits + 7) / 8 > TPCZSHDR::TPC_ZS_PAGE_SIZE) { - seqLen = ((TPCZSHDR::TPC_ZS_PAGE_SIZE - sizeChk) * 8 - streamSizeChkBits) / encodeBits; - } - // sizeChk += (seqLen * encodeBits + streamSizeChkBits + 7) / 8; - // printf("Endpoint %d (%d), Pos %d, Chk %d, Len %d, rows %d, StreamSize %d %d, time %d (%d), row %d (%d), pad %d\n", endpoint, lastEndpoint, (int) (pagePtr - reinterpret_cast(page)), sizeChk, seqLen, nRowsInTB, streamSize8, streamSize, (int) ZSEncoderGetTime(tmpBuffer[k]), lastTime, (int) ZSEncoderGetRow(tmpBuffer[k]), lastRow, ZSEncoderGetPad(tmpBuffer[k])); - } - } else { - nexthbf = -1; - } - if (k >= tmpBuffer.size() || endpoint != lastEndpoint || ZSEncoderGetTime(tmpBuffer[k]) != lastTime) { - if (pagePtr != reinterpret_cast(page)) { - pagePtr += 2 * nRowsInTB; - ZSstreamOut(streamBuffer.data(), streamSize, streamBuffer8.data(), streamSize8, encodeBits); - pagePtr = std::copy(streamBuffer8.data(), streamBuffer8.data() + streamSize8, pagePtr); - if (pagePtr - reinterpret_cast(page) > 8192) { - throw std::runtime_error("internal error during ZS encoding"); - } - streamSize8 = 0; - for (int l = 1; l < nRowsInTB; l++) { - tbHdr->rowAddr1()[l - 1] += 2 * nRowsInTB; - } - } - if (page && (k >= tmpBuffer.size() || endpoint != lastEndpoint)) { - const rdh_utils::FEEIDType rawfeeid = rdh_utils::getFEEID(rawcru, rawendpoint, rawlnk); - size_t size = (padding || lastEndpoint == -1 || hbf == nexthbf) ? TPCZSHDR::TPC_ZS_PAGE_SIZE : (pagePtr - (unsigned char*)page); - size = CAMath::nextMultipleOf(size); -#ifdef GPUCA_O2_LIB - if (raw) { - raw->addData(rawfeeid, rawcru, rawlnk, rawendpoint, *ir + hbf * o2::constants::lhc::LHCMaxBunches, gsl::span((char*)page + sizeof(o2::header::RAWDataHeader), (char*)page + size), true); - } else -#endif - { - o2::header::RAWDataHeader* rdh = (o2::header::RAWDataHeader*)page; - o2::raw::RDHUtils::setHeartBeatOrbit(*rdh, hbf + orbitShift); - o2::raw::RDHUtils::setHeartBeatBC(*rdh, bcShiftInFirstHBF); - o2::raw::RDHUtils::setMemorySize(*rdh, size); - o2::raw::RDHUtils::setVersion(*rdh, o2::raw::RDHUtils::getVersion()); - o2::raw::RDHUtils::setFEEID(*rdh, rawfeeid); - } - } - if (k >= tmpBuffer.size()) { - break; - } - } - if (endpoint != lastEndpoint) { - if (raw) { - page = &singleBuffer; - } else { - if (buffer[i][endpoint].size() == 0 && nexthbf > orbitShift) { - // Emplace empty page with RDH containing beginning of TF - buffer[i][endpoint].emplace_back(); - ZSfillEmpty(&buffer[i][endpoint].back(), bcShiftInFirstHBF, rdh_utils::getFEEID(i * 10 + endpoint / 2, endpoint & 1, rawlnk), orbitShift); - totalPages++; - } - buffer[i][endpoint].emplace_back(); - page = &buffer[i][endpoint].back(); - } - hbf = nexthbf; - pagePtr = reinterpret_cast(page); - std::fill(page->begin(), page->end(), 0); - pagePtr += sizeof(o2::header::RAWDataHeader); - hdr = reinterpret_cast(pagePtr); - pagePtr += sizeof(*hdr); - hdr->version = zs12bit ? 2 : 1; - hdr->cruID = i * 10 + region; - rawcru = i * 10 + region; - rawendpoint = endpoint & 1; - hdr->timeOffset = (long)ZSEncoderGetTime(tmpBuffer[k]) * LHCBCPERTIMEBIN - (long)hbf * o2::constants::lhc::LHCMaxBunches; - lastTime = -1; - tbHdr = nullptr; - lastEndpoint = endpoint; - totalPages++; - } - if (ZSEncoderGetTime(tmpBuffer[k]) != lastTime) { - if (lastTime != -1) { - hdr->nTimeBins += ZSEncoderGetTime(tmpBuffer[k]) - lastTime - 1; - pagePtr += (ZSEncoderGetTime(tmpBuffer[k]) - lastTime - 1) * 2; - } - hdr->nTimeBins++; - lastTime = ZSEncoderGetTime(tmpBuffer[k]); - if ((pagePtr - reinterpret_cast(page)) & 1) { - pagePtr++; - } - tbHdr = reinterpret_cast(pagePtr); - tbHdr->rowMask |= (endpoint & 1) << 15; - nRowsInTB = 0; - lastRow = GPUCA_ROW_COUNT; - } - if (ZSEncoderGetRow(tmpBuffer[k]) != lastRow) { - tbHdr->rowMask |= 1 << (ZSEncoderGetRow(tmpBuffer[k]) - endpointStart); - lastRow = ZSEncoderGetRow(tmpBuffer[k]); - ZSstreamOut(streamBuffer.data(), streamSize, streamBuffer8.data(), streamSize8, encodeBits); - if (nRowsInTB) { - tbHdr->rowAddr1()[nRowsInTB - 1] = (pagePtr - reinterpret_cast(page)) + streamSize8; - } - nRowsInTB++; - nSeq = streamBuffer8.data() + streamSize8++; - *nSeq = 0; - } - (*nSeq)++; - streamBuffer8[streamSize8++] = ZSEncoderGetPad(tmpBuffer[k]); - streamBuffer8[streamSize8++] = streamSize + seqLen; - hdr->nADCsamples += seqLen; - for (int l = 0; l < seqLen; l++) { - streamBuffer[streamSize++] = (unsigned short)(ZSEncoderGetCharge(tmpBuffer[k + l]) * encodeBitsFactor + 0.5f); - } - k += seqLen - 1; - } - if (!raw) { - for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { - if (buffer[i][j].size() == 0) { - buffer[i][j].emplace_back(); - ZSfillEmpty(&buffer[i][j].back(), bcShiftInFirstHBF, rdh_utils::getFEEID(i * 10 + j / 2, j & 1, rawlnk), orbitShift); - totalPages++; - } - } - } + + zsEncoderRun enc{{{.iSector = i, .raw = raw, .ir = ir, .param = param, .padding = padding}}}; + enc.encodeBits = zs12bit ? TPCZSHDR::TPC_ZS_NBITS_V2 : TPCZSHDR::TPC_ZS_NBITS_V1; + enc.zsVersion = zs12bit ? 2 : 1; + totalPages += enc.run(buffer[i], tmpBuffer); // Verification if (verify) { - std::vector compareBuffer; - compareBuffer.reserve(tmpBuffer.size()); - for (unsigned int j = 0; j < GPUTrackingInOutZS::NENDPOINTS; j++) { - unsigned int firstOrbit = o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)buffer[i][j].data()); - for (unsigned int k = 0; k < buffer[i][j].size(); k++) { - page = &buffer[i][j][k]; - pagePtr = reinterpret_cast(page); - const o2::header::RAWDataHeader* rdh = (const o2::header::RAWDataHeader*)pagePtr; - if (o2::raw::RDHUtils::getMemorySize(*rdh) == sizeof(o2::header::RAWDataHeader)) { - continue; - } - pagePtr += sizeof(o2::header::RAWDataHeader); - hdr = reinterpret_cast(pagePtr); - pagePtr += sizeof(*hdr); - if (hdr->version != 1 && hdr->version != 2) { - throw std::runtime_error("invalid ZS version"); - } - const bool decode12bit = hdr->version == 2; - const unsigned int decodeBits = decode12bit ? TPCZSHDR::TPC_ZS_NBITS_V2 : TPCZSHDR::TPC_ZS_NBITS_V1; - const float decodeBitsFactor = 1.f / (1 << (decodeBits - 10)); - unsigned int mask = (1 << decodeBits) - 1; - int cruid = hdr->cruID; - unsigned int sector = cruid / 10; - if (sector != i) { - throw std::runtime_error("invalid TPC sector"); - } - region = cruid % 10; - if ((unsigned int)region != j / 2) { - throw std::runtime_error("CRU ID / endpoint mismatch"); - } - int nRowsRegion = param.tpcGeometry.GetRegionRows(region); - - int timeBin = (hdr->timeOffset + (unsigned long)(o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) - firstOrbit) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN; - for (int l = 0; l < hdr->nTimeBins; l++) { - if ((pagePtr - reinterpret_cast(page)) & 1) { - pagePtr++; - } - tbHdr = reinterpret_cast(pagePtr); - bool upperRows = tbHdr->rowMask & 0x8000; - if (tbHdr->rowMask != 0 && ((upperRows) ^ ((j & 1) != 0))) { - throw std::runtime_error("invalid endpoint"); - } - const int rowOffset = param.tpcGeometry.GetRegionStart(region) + (upperRows ? (nRowsRegion / 2) : 0); - const int nRows = upperRows ? (nRowsRegion - nRowsRegion / 2) : (nRowsRegion / 2); - const int nRowsUsed = __builtin_popcount((unsigned int)(tbHdr->rowMask & 0x7FFF)); - pagePtr += nRowsUsed ? (2 * nRowsUsed) : 2; - int rowPos = 0; - for (int m = 0; m < nRows; m++) { - if ((tbHdr->rowMask & (1 << m)) == 0) { - continue; - } - unsigned char* rowData = rowPos == 0 ? pagePtr : (reinterpret_cast(page) + tbHdr->rowAddr1()[rowPos - 1]); - const int nSeqRead = *rowData; - unsigned char* adcData = rowData + 2 * nSeqRead + 1; - int nADC = (rowData[2 * nSeqRead] * decodeBits + 7) / 8; - pagePtr += 1 + 2 * nSeqRead + nADC; - unsigned int byte = 0, bits = 0, pos10 = 0; - for (int n = 0; n < nADC; n++) { - byte |= *(adcData++) << bits; - bits += 8; - while (bits >= decodeBits) { - streamBuffer[pos10++] = byte & mask; - byte = byte >> decodeBits; - bits -= decodeBits; - } - } - pos10 = 0; - for (int n = 0; n < nSeqRead; n++) { - const int seqLen = rowData[(n + 1) * 2] - (n ? rowData[n * 2] : 0); - for (int o = 0; o < seqLen; o++) { - compareBuffer.emplace_back(o2::tpc::Digit{0, (float)streamBuffer[pos10++] * decodeBitsFactor, (tpccf::Row)(rowOffset + m), (tpccf::Pad)(rowData[n * 2 + 1] + o), timeBin + l}); - } - } - rowPos++; - } - } - } - } - for (unsigned int j = 0; j < tmpBuffer.size(); j++) { - const unsigned int decodeBits = zs12bit ? TPCZSHDR::TPC_ZS_NBITS_V2 : TPCZSHDR::TPC_ZS_NBITS_V1; - const float decodeBitsFactor = (1 << (decodeBits - 10)); - const float c = zs12bit ? (float)((int)(ZSEncoderGetCharge(tmpBuffer[j]) * decodeBitsFactor + 0.5f)) / decodeBitsFactor : (float)(int)(ZSEncoderGetCharge(tmpBuffer[j]) + 0.5f); - int ok = c == ZSEncoderGetCharge(compareBuffer[j]) && (int)ZSEncoderGetTime(tmpBuffer[j]) == (int)ZSEncoderGetTime(compareBuffer[j]) && (int)ZSEncoderGetPad(tmpBuffer[j]) == (int)ZSEncoderGetPad(compareBuffer[j]) && (int)ZSEncoderGetRow(tmpBuffer[j]) == (int)ZSEncoderGetRow(compareBuffer[j]); - if (ok) { - continue; - } - nErrors++; - printf("%4u: OK %d: Charge %3d %3d Time %4d %4d Pad %3d %3d Row %3d %3d\n", j, ok, - (int)c, (int)ZSEncoderGetCharge(compareBuffer[j]), (int)ZSEncoderGetTime(tmpBuffer[j]), (int)ZSEncoderGetTime(compareBuffer[j]), (int)ZSEncoderGetPad(tmpBuffer[j]), (int)ZSEncoderGetPad(compareBuffer[j]), (int)ZSEncoderGetRow(tmpBuffer[j]), (int)ZSEncoderGetRow(compareBuffer[j])); - } + nErrors += enc.compare(buffer[i], tmpBuffer); } } @@ -553,11 +671,18 @@ void GPUReconstructionConvert::RunZSEncoder(const S& in, std::unique_ptr(const GPUTrackingInOutDigits&, std::unique_ptr*, unsigned int*, o2::raw::RawFileWriter*, const o2::InteractionRecord*, const GPUParam&, bool, bool, float, bool); +#ifdef GPUCA_O2_LIB +template void GPUReconstructionConvert::RunZSEncoder(const DigitArray&, std::unique_ptr*, unsigned int*, o2::raw::RawFileWriter*, const o2::InteractionRecord*, const GPUParam&, bool, bool, float, bool); +#endif +#endif + void GPUReconstructionConvert::RunZSEncoderCreateMeta(const unsigned long long int* buffer, const unsigned int* sizes, void** ptrs, GPUTrackingInOutZS* out) { unsigned long long int offset = 0; diff --git a/GPU/GPUTracking/Base/GPUReconstructionConvert.h b/GPU/GPUTracking/Base/GPUReconstructionConvert.h index dc4dbcb91860d..9f0a4e9a69f8b 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionConvert.h +++ b/GPU/GPUTracking/Base/GPUReconstructionConvert.h @@ -51,17 +51,13 @@ class GPUReconstructionConvert constexpr static unsigned int NSLICES = GPUCA_NSLICES; static void ConvertNativeToClusterData(o2::tpc::ClusterNativeAccess* native, std::unique_ptr* clusters, unsigned int* nClusters, const TPCFastTransform* transform, int continuousMaxTimeBin = 0); static void ConvertRun2RawToNative(o2::tpc::ClusterNativeAccess& native, std::unique_ptr& nativeBuffer, const AliHLTTPCRawCluster** rawClusters, unsigned int* nRawClusters); - template + template static void RunZSEncoder(const S& in, std::unique_ptr* outBuffer, unsigned int* outSizes, o2::raw::RawFileWriter* raw, const o2::InteractionRecord* ir, const GPUParam& param, bool zs12bit, bool verify, float threshold = 0.f, bool padding = false); static void RunZSEncoderCreateMeta(const unsigned long long int* buffer, const unsigned int* sizes, void** ptrs, GPUTrackingInOutZS* out); static void RunZSFilter(std::unique_ptr* buffers, const o2::tpc::Digit* const* ptrs, size_t* nsb, const size_t* ns, const GPUParam& param, bool zs12bit, float threshold); static int GetMaxTimeBin(const o2::tpc::ClusterNativeAccess& native); static int GetMaxTimeBin(const GPUTrackingInOutDigits& digits); static int GetMaxTimeBin(const GPUTrackingInOutZS& zspages); - - private: - static void ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits); - static void ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int orbit); }; } // namespace gpu } // namespace GPUCA_NAMESPACE diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index dd0225448c21a..2f221650042fc 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -130,6 +130,7 @@ std::pair GPUChainTracking::TPCClusterizerDecodeZSCo continue; } const TPCZSHDR* const hdr = (const TPCZSHDR*)(page + sizeof(o2::header::RAWDataHeader)); + mCFContext->zsVersion = hdr->version; nDigits += hdr->nADCsamples; unsigned int timeBin = (hdr->timeOffset + (o2::raw::RDHUtils::getHeartBeatOrbit(*rdh) - firstHBF) * o2::constants::lhc::LHCMaxBunches) / LHCBCPERTIMEBIN; if (timeBin + hdr->nTimeBins > mCFContext->tpcMaxTimeBin) { @@ -524,7 +525,11 @@ int GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) if (mIOPtrs.tpcZS) { int firstHBF = (mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit) ? mIOPtrs.settingsTF->tfStartOrbit : (mIOPtrs.tpcZS->slice[iSlice].count[0] && mIOPtrs.tpcZS->slice[iSlice].nZSPtr[0][0]) ? o2::raw::RDHUtils::getHeartBeatOrbit(*(const o2::header::RAWDataHeader*)mIOPtrs.tpcZS->slice[iSlice].zsPtr[0][0]) : 0; - runKernel(GetGridBlk(doGPU ? clusterer.mPmemory->counters.nPagesSubslice : GPUTrackingInOutZS::NENDPOINTS, lane), {iSlice}, {}, firstHBF); + if (mCFContext->zsVersion >= 1 || mCFContext->zsVersion <= 2) { + runKernel(GetGridBlk(doGPU ? clusterer.mPmemory->counters.nPagesSubslice : GPUTrackingInOutZS::NENDPOINTS, lane), {iSlice}, {}, firstHBF); + } else { + GPUFatal("Data with invalid TPC ZS mode received"); + } TransferMemoryResourceLinkToHost(RecoStep::TPCClusterFinding, clusterer.mMemoryId, lane); } } diff --git a/GPU/GPUTracking/Global/GPUChainTrackingTransformation.cxx b/GPU/GPUTracking/Global/GPUChainTrackingTransformation.cxx index 94d4d8252b7f5..aaa233fd3cdac 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingTransformation.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingTransformation.cxx @@ -129,7 +129,7 @@ void GPUChainTracking::ConvertZSEncoder(bool zs12bit) mIOMem.tpcZSmeta2.reset(new GPUTrackingInOutZS::GPUTrackingInOutZSMeta); mIOMem.tpcZSmeta.reset(new GPUTrackingInOutZS); o2::InteractionRecord ir{0, mIOPtrs.settingsTF && mIOPtrs.settingsTF->hasTfStartOrbit ? mIOPtrs.settingsTF->tfStartOrbit : 0u}; - GPUReconstructionConvert::RunZSEncoder(*mIOPtrs.tpcPackedDigits, &mIOMem.tpcZSpages, &mIOMem.tpcZSmeta2->n[0][0], nullptr, &ir, param(), zs12bit, true); + GPUReconstructionConvert::RunZSEncoder(*mIOPtrs.tpcPackedDigits, &mIOMem.tpcZSpages, &mIOMem.tpcZSmeta2->n[0][0], nullptr, &ir, param(), zs12bit, true); GPUReconstructionConvert::RunZSEncoderCreateMeta(mIOMem.tpcZSpages.get(), &mIOMem.tpcZSmeta2->n[0][0], &mIOMem.tpcZSmeta2->ptr[0][0], mIOMem.tpcZSmeta.get()); mIOPtrs.tpcZS = mIOMem.tpcZSmeta.get(); if (GetProcessingSettings().registerStandaloneInputMemory) { diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChainContext.h b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChainContext.h index b52a67e2dee6a..58ddbef5d09dc 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChainContext.h +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFChainContext.h @@ -41,6 +41,7 @@ struct GPUTPCCFChainContext { unsigned char* zsDevice; }; + unsigned int zsVersion; std::vector fragmentData; unsigned int nPagesTotal; unsigned int nPagesSectorMax;