diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index f982e0bfd..7d5803181 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -118,7 +118,8 @@ const ControlInfoMap::Map ipaAfControls{ /* Platform specific controls */ const std::map platformControls { { "pisp", { - { &controls::rpi::ScalerCrops, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) } + { &controls::rpi::ScalerCrops, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) }, + { &controls::rpi::OutputWindows, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) }, } }, }; @@ -291,6 +292,11 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa } } + /* Platform specific controls should be available after configure() too. */ + auto platformCtrlsIt = platformControls.find(controller_.getTarget()); + if (platformCtrlsIt != platformControls.end()) + ctrlMap.merge(ControlInfoMap::Map(platformCtrlsIt->second)); + result->controlInfo = ControlInfoMap(std::move(ctrlMap), controls::controls); return platformConfigure(params, result); @@ -1316,6 +1322,11 @@ void IpaBase::applyControls(const ControlList &controls) break; } + case controls::rpi::OUTPUT_WINDOWS: { + /* The IPA does nothing with this, but avoid the warning. */ + break; + } + case controls::FRAME_DURATION_LIMITS: { auto frameDurations = ctrl.second.get>(); applyFrameDurations(frameDurations[0] * 1.0us, frameDurations[1] * 1.0us); diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 9d0f30002..20113e208 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -59,6 +59,29 @@ controls: \sa ScalerCrop + - OutputWindows: + type: Rectangle + size: [n] + direction: inout + description: | + An array of rectangles, where each defines a region within the output + image for the stream corresponding to the position in this array. + + The camera image is resized and shifted to occupy only this + particular window within the overall output image, and is intended + for applications that want the camera image to be (for example) + "letterboxed" into a larger output (such as a number of machine + learning applications). + + There are platform specific limits on how small these windows can be, + and in particular format specific alignment constraints will apply to + the horizontal offset of the window. + + Unless changed, the output windows will occupy the entire image + buffers, in the usual manner. + + The OutputWindows control is supported only on the Pi5/PiSP platform. + - PispStatsOutput: type: uint8_t direction: out diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 5a5acf6a1..cb6c9f236 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -580,17 +580,22 @@ int PipelineHandlerBase::configure(Camera *camera, CameraConfiguration *config) Rectangle ispMinCrop = data->scaleIspCrop(Rectangle(cropParams.ispMinCropSize)); ctrlMap[&controls::ScalerCrop] = ControlInfo(ispMinCrop, data->sensorInfo_.analogCrop, data->scaleIspCrop(cropParams.ispCrop)); - if (data->cropParams_.size() == 2) { - /* - * The control map for rpi::ScalerCrops has the min value - * as the default crop for stream 0, max value as the default - * value for stream 1. - */ - ctrlMap[&controls::rpi::ScalerCrops] = - ControlInfo(data->scaleIspCrop(data->cropParams_.at(0).ispCrop), - data->scaleIspCrop(data->cropParams_.at(1).ispCrop), - ctrlMap[&controls::ScalerCrop].def()); - } + + /* Always advertise the ScalerCrops control, even if there's only one stream. */ + Rectangle streamTwoDefault{ 65535, 65535, 65535, 65535 }; + + if (data->cropParams_.size() == 2) + streamTwoDefault = data->scaleIspCrop(data->cropParams_.at(1).ispCrop); + + /* + * The control map for rpi::ScalerCrops has the min value + * as the default crop for stream 0, max value as the default + * value for stream 1 (where a second stream exists). + */ + ctrlMap[&controls::rpi::ScalerCrops] = + ControlInfo(data->scaleIspCrop(data->cropParams_.at(0).ispCrop), + streamTwoDefault, + ctrlMap[&controls::ScalerCrop].def()); } data->controlInfo_ = ControlInfoMap(std::move(ctrlMap), result.controlInfo.idmap()); @@ -654,7 +659,7 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) /* Check if a ScalerCrop control was specified. */ if (controls) - data->applyScalerCrop(*controls); + data->applyScalerCropAndWindows(*controls); /* Start the IPA. */ ipa::RPi::StartResult result; @@ -1314,7 +1319,7 @@ Rectangle CameraData::scaleIspCrop(const Rectangle &ispCrop) const return nativeCrop; } -void CameraData::applyScalerCrop(const ControlList &controls) +void CameraData::applyScalerCropAndWindows(const ControlList &controls) { const auto &scalerCropRPi = controls.get>(controls::rpi::ScalerCrops); const auto &scalerCropCore = controls.get(controls::ScalerCrop); @@ -1361,7 +1366,32 @@ void CameraData::applyScalerCrop(const ControlList &controls) if (ispCrop != cropParams_.at(i).ispCrop) { cropParams_.at(i).ispCrop = ispCrop; - platformSetIspCrop(cropParams_.at(i).ispIndex, ispCrop); + platformSetIspCrop(ispIndices_[i], ispCrop); + } + } + + const auto &outputWindows = controls.get>(controls::rpi::OutputWindows); + if (outputWindows) { + for (unsigned int i = 0; i < outputWindows->size() && i < outputWindows_.size(); i++) { + const Rectangle &outputWindow = outputWindows->data()[i]; + const StreamConfiguration &streamConfig = streams_[i]->configuration(); + + /* Do some basic checks here; platformSetOutputWindow will do the rest. */ + if (outputWindow.x + outputWindow.width > streamConfig.size.width) + LOG(RPI, Warning) << "Offset " << outputWindow.x << " plus width " + << outputWindow.width << " exceed maximum " + << streamConfig.size.width; + else if (outputWindow.y + outputWindow.height > streamConfig.size.height) + LOG(RPI, Warning) << "Offset " << outputWindow.y << " plus height " + << outputWindow.height << " exceed maximum " + << streamConfig.size.height; + else { + LOG(RPI, Debug) << "Ouptut window " << i << " set to " + << outputWindow.width << " x " << outputWindow.height; + /* platformSetOutputWindow will issue a warning if it fails. */ + if (platformSetOutputWindow(i, outputWindow) == 0) + outputWindows_[i] = outputWindow; + } } } } @@ -1530,6 +1560,11 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request crops.size())); } } + + if (outputWindows_.size()) + request->_d()->metadata().set(controls::rpi::OutputWindows, + Span(outputWindows_.data(), + outputWindows_.size())); } static bool isControlDelayed(unsigned int id) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 758155ee0..ad3b742ab 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -82,8 +82,9 @@ class CameraData : public Camera::Private void setSensorControls(ControlList &controls); Rectangle scaleIspCrop(const Rectangle &ispCrop) const; - void applyScalerCrop(const ControlList &controls); - virtual void platformSetIspCrop(unsigned int index, const Rectangle &ispCrop) = 0; + void applyScalerCropAndWindows(const ControlList &controls); + virtual void platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop) = 0; + virtual int platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow) = 0; void cameraTimeout(); void frameStarted(uint32_t sequence); @@ -135,8 +136,8 @@ class CameraData : public Camera::Private IPACameraSensorInfo sensorInfo_; struct CropParams { - CropParams(Rectangle ispCrop_, Size ispMinCropSize_, unsigned int ispIndex_) - : ispCrop(ispCrop_), ispMinCropSize(ispMinCropSize_), ispIndex(ispIndex_) + CropParams(Rectangle ispCrop_, Size ispMinCropSize_) + : ispCrop(ispCrop_), ispMinCropSize(ispMinCropSize_) { } @@ -144,13 +145,20 @@ class CameraData : public Camera::Private Rectangle ispCrop; /* Minimum crop size in ISP output pixels */ Size ispMinCropSize; - /* Index of the ISP output channel for this crop */ - unsigned int ispIndex; }; + /* + * Vector recording the ISP index (output branch number) for the streams, ordered + * according to the stream's position in the CameraConfiguration. + */ + std::vector ispIndices_; + /* Mapping of CropParams keyed by the output stream order in CameraConfiguration */ std::map cropParams_; + /* Vector of output windows, again ordered according to the CameraConfiguration. */ + std::vector outputWindows_; + unsigned int startupFrameCount_; unsigned int invalidFrameCount_; diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index c9d89d58b..bdc609072 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -51,6 +51,11 @@ enum class Cfe : unsigned int { Output0, Embedded, Stats, Config }; enum class Isp : unsigned int { Input, Output0, Output1, TdnInput, TdnOutput, StitchInput, StitchOutput, Config }; +struct OutputInfo { + unsigned int minWidth; + unsigned int alignment; +}; + /* Offset for all compressed buffers; mode for TDN and Stitch. */ constexpr unsigned int DefaultCompressionOffset = 2048; constexpr unsigned int DefaultCompressionMode = 1; @@ -691,12 +696,15 @@ unsigned int getFormatAlignment(const V4L2PixelFormat &fourcc) /* Calculate the amount of software downscale required (which is a power of 2). */ unsigned int calculateSwDownscale(const V4L2DeviceFormat &format, unsigned int largestWidth, - unsigned int platformMaxDownscale) + unsigned int platformMaxDownscale, OutputInfo *outputInfo = nullptr) { unsigned int formatAlignment = getFormatAlignment(format.fourcc); unsigned int maxDownscale = platformMaxDownscale * 16 / formatAlignment; unsigned int limitWidth = largestWidth / maxDownscale; + if (outputInfo) + *outputInfo = { limitWidth, formatAlignment }; + unsigned int hwWidth = format.size.width; unsigned int swDownscale = 1; for (; hwWidth < limitWidth; hwWidth *= 2, swDownscale *= 2); @@ -814,7 +822,8 @@ class PiSPCameraData final : public RPi::CameraData bool calculateCscConfiguration(const V4L2DeviceFormat &v4l2Format, pisp_be_ccm_config &csc); int configureBe(const std::optional &yuvColorSpace); - void platformSetIspCrop(unsigned int index, const Rectangle &ispCrop) override; + void platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop) override; + int platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow) override; void prepareCfe(); void prepareBe(uint32_t bufferId, bool stitchSwapBuffers); @@ -842,6 +851,8 @@ class PiSPCameraData final : public RPi::CameraData } std::string last_dump_file_; + + std::vector outputInfo_; }; class PipelineHandlerPiSP : public RPi::PipelineHandlerBase @@ -1505,7 +1516,10 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf s == &isp_[Isp::Output1]; }), streams_.end()); + ispIndices_.clear(); cropParams_.clear(); + outputWindows_.clear(); + for (unsigned int i = 0; i < outStreams.size(); i++) { StreamConfiguration *cfg = outStreams[i].cfg; unsigned int ispIndex; @@ -1524,6 +1538,7 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf beEnables |= PISP_BE_RGB_ENABLE_OUTPUT0; ispIndex = 0; } + ispIndices_.push_back(ispIndex); format = outStreams[i].format; bool needs32BitConversion = adjustDeviceFormat(format); @@ -1535,8 +1550,10 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf PixelFormat pixFmt = format.fourcc.toPixelFormat(); /* If there's excessive downscaling we'll do some of it in software. */ + outputInfo_.emplace_back(OutputInfo{ 0, 0 }); unsigned int swDownscale = calculateSwDownscale(format, largestWidth, - be_->GetMaxDownscale()); + be_->GetMaxDownscale(), + &outputInfo_.back()); unsigned int hwWidth = format.size.width * swDownscale; format.size.width = hwWidth; @@ -1566,6 +1583,8 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf if (needs32BitConversion) flags |= StreamFlag::Needs32bitConv; + outputWindows_.push_back(Rectangle(outStreams[i].cfg->size)); + /* Set smallest selection the ISP will allow. */ Size minCrop{ 32, 32 }; @@ -1590,7 +1609,7 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf */ cropParams_.emplace(std::piecewise_construct, std::forward_as_tuple(outStreams[i].index), - std::forward_as_tuple(ispCrop, minCrop, ispIndex)); + std::forward_as_tuple(ispCrop, minCrop)); cfg->setStream(stream); stream->setFlags(flags); @@ -2138,7 +2157,7 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) return 0; } -void PiSPCameraData::platformSetIspCrop(unsigned int index, const Rectangle &ispCrop) +void PiSPCameraData::platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop) { pisp_be_crop_config beCrop = { static_cast(ispCrop.x), @@ -2147,8 +2166,55 @@ void PiSPCameraData::platformSetIspCrop(unsigned int index, const Rectangle &isp static_cast(ispCrop.height) }; - LOG(RPI, Debug) << "Output " << index << " " << ispCrop.toString(); - be_->SetCrop(index, beCrop); + LOG(RPI, Debug) << "Output " << ispIndex << " " << ispCrop.toString(); + be_->SetCrop(ispIndex, beCrop); +} + +int PiSPCameraData::platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow) +{ + pisp_be_output_format_config config; + pisp_be_output_format_extra extra; + BackEnd::SmartResize resize; + const OutputInfo &outputInfo = outputInfo_[streamIndex]; + unsigned int ispIndex = ispIndices_[streamIndex]; + + be_->GetOutputFormat(ispIndex, config, extra); + + /* + * We insist on a minimum size that ensures no software downscaling is ever required, + * which limits the amount of reconfiguration one might have to do here. + */ + if (outputWindow.width < outputInfo.minWidth) { + LOG(RPI, Warning) << "Window width " << outputWindow.width + << " less than minimum value " << outputInfo.minWidth; + return -1; + } else if (outputWindow.height < 2) { + LOG(RPI, Warning) << "Window height " << outputWindow.height + << " less than minimum value 2"; + return -1; + } else if (outputWindow.x % outputInfo.alignment) { + LOG(RPI, Warning) << "Window x offset " << outputWindow.x + << " does not match pixel alignment " << outputInfo.alignment; + return -1; + } else if (outputWindow.y & 1) { + LOG(RPI, Warning) << "Window y offset " << outputWindow.y + << " must be even"; + return -1; + } + + resize.width = outputWindow.width; + resize.height = outputWindow.height; + be_->SetSmartResize(ispIndex, resize); + + config.image.width = outputWindow.width; + config.image.height = outputWindow.height; + + extra.offset_x = outputWindow.x; + extra.offset_y = outputWindow.y; + + be_->SetOutputFormat(ispIndex, config, extra); + + return 0; } int PiSPCameraData::platformInitIpa(ipa::RPi::InitParams ¶ms) @@ -2307,7 +2373,7 @@ void PiSPCameraData::tryRunPipeline() ASSERT(request->metadata().empty()); /* See if a new ScalerCrop value needs to be applied. */ - applyScalerCrop(request->controls()); + applyScalerCropAndWindows(request->controls()); fillRequestMetadata(job.sensorControls, request); diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index ac86a43ad..374be9298 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -121,11 +121,17 @@ class Vc4CameraData final : public RPi::CameraData Config config_; private: - void platformSetIspCrop([[maybe_unused]] unsigned int index, const Rectangle &ispCrop) override + void platformSetIspCrop([[maybe_unused]] unsigned int ispIndex, const Rectangle &ispCrop) override { Rectangle crop = ispCrop; isp_[Isp::Input].dev()->setSelection(V4L2_SEL_TGT_CROP, &crop); } + int platformSetOutputWindow([[maybe_unused]] unsigned int streamIndex, + [[maybe_unused]] const Rectangle &outputWindow) override + { + LOG(RPI, Warning) << "VC4 platform does not support setting output windows"; + return -1; + } int platformConfigure(const RPi::RPiCameraConfiguration *rpiConfig) override; int platformConfigureIpa(ipa::RPi::ConfigParams ¶ms) override; @@ -756,7 +762,7 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi */ cropParams_.emplace(std::piecewise_construct, std::forward_as_tuple(0), - std::forward_as_tuple(ispCrop, testCrop.size(), 0)); + std::forward_as_tuple(ispCrop, testCrop.size())); return 0; } @@ -980,7 +986,7 @@ void Vc4CameraData::tryRunPipeline() ASSERT(request->metadata().empty()); /* See if a new ScalerCrop value needs to be applied. */ - applyScalerCrop(request->controls()); + applyScalerCropAndWindows(request->controls()); fillRequestMetadata(bayerFrame.controls, request);