Add a new OutputWindows control

src/ipa/rpi/common/ipa_base.cpp

@@ -118,7 +118,8 @@ const ControlInfoMap::Map ipaAfControls{
 /* Platform specific controls */
 const std::map<const std::string, ControlInfoMap::Map> platformControls {
 	{ "pisp", {
-		{ &controls::rpi::ScalerCrops, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) }
+			  { &controls::rpi::ScalerCrops, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },
+			  { &controls::rpi::OutputWindows, ControlInfo(Rectangle{}, Rectangle(65535, 65535, 65535, 65535), Rectangle{}) },
 	} },
 };
 
@@ -291,6 +292,11 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa
 		}
 	}
 
+	/* Platform specific controls should be available after configure() too. */
+	auto platformCtrlsIt = platformControls.find(controller_.getTarget());
+	if (platformCtrlsIt != platformControls.end())
+		ctrlMap.merge(ControlInfoMap::Map(platformCtrlsIt->second));
+
 	result->controlInfo = ControlInfoMap(std::move(ctrlMap), controls::controls);
 
 	return platformConfigure(params, result);
@@ -1316,6 +1322,11 @@ void IpaBase::applyControls(const ControlList &controls)
 			break;
 		}
 
+		case controls::rpi::OUTPUT_WINDOWS: {
+			/* The IPA does nothing with this, but avoid the warning. */
+			break;
+		}
+
 		case controls::FRAME_DURATION_LIMITS: {
 			auto frameDurations = ctrl.second.get<Span<const int64_t>>();
 			applyFrameDurations(frameDurations[0] * 1.0us, frameDurations[1] * 1.0us);

src/libcamera/control_ids_rpi.yaml

@@ -59,6 +59,29 @@ controls:
 
         \sa ScalerCrop
 
+  - OutputWindows:
+      type: Rectangle
+      size: [n]
+      direction: inout
+      description: |
+        An array of rectangles, where each defines a region within the output
+        image for the stream corresponding to the position in this array.
+
+        The camera image is resized and shifted to occupy only this
+        particular window within the overall output image, and is intended
+        for applications that want the camera image to be (for example)
+        "letterboxed" into a larger output (such as a number of machine
+        learning applications).
+
+        There are platform specific limits on how small these windows can be,
+        and in particular format specific alignment constraints will apply to
+        the horizontal offset of the window.
+
+        Unless changed, the output windows will occupy the entire image
+        buffers, in the usual manner.
+
+        The OutputWindows control is supported only on the Pi5/PiSP platform.
+
   - PispStatsOutput:
       type: uint8_t
       direction: out

src/libcamera/pipeline/rpi/common/pipeline_base.cpp

@@ -580,17 +580,22 @@ int PipelineHandlerBase::configure(Camera *camera, CameraConfiguration *config)
 		Rectangle ispMinCrop = data->scaleIspCrop(Rectangle(cropParams.ispMinCropSize));
 		ctrlMap[&controls::ScalerCrop] = ControlInfo(ispMinCrop, data->sensorInfo_.analogCrop,
 							     data->scaleIspCrop(cropParams.ispCrop));
-		if (data->cropParams_.size() == 2) {
-			/*
-			 * The control map for rpi::ScalerCrops has the min value
-			 * as the default crop for stream 0, max value as the default
-			 * value for stream 1.
-			 */
-			ctrlMap[&controls::rpi::ScalerCrops] =
-				ControlInfo(data->scaleIspCrop(data->cropParams_.at(0).ispCrop),
-					    data->scaleIspCrop(data->cropParams_.at(1).ispCrop),
-					    ctrlMap[&controls::ScalerCrop].def());
-		}
+
+		/* Always advertise the ScalerCrops control, even if there's only one stream. */
+		Rectangle streamTwoDefault{ 65535, 65535, 65535, 65535 };
+
+		if (data->cropParams_.size() == 2)
+			streamTwoDefault = data->scaleIspCrop(data->cropParams_.at(1).ispCrop);
+
+		/*
+		 * The control map for rpi::ScalerCrops has the min value
+		 * as the default crop for stream 0, max value as the default
+		 * value for stream 1 (where a second stream exists).
+		 */
+		ctrlMap[&controls::rpi::ScalerCrops] =
+			ControlInfo(data->scaleIspCrop(data->cropParams_.at(0).ispCrop),
+				    streamTwoDefault,
+				    ctrlMap[&controls::ScalerCrop].def());
 	}
 
 	data->controlInfo_ = ControlInfoMap(std::move(ctrlMap), result.controlInfo.idmap());
@@ -654,7 +659,7 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls)
 
 	/* Check if a ScalerCrop control was specified. */
 	if (controls)
-		data->applyScalerCrop(*controls);
+		data->applyScalerCropAndWindows(*controls);
 
 	/* Start the IPA. */
 	ipa::RPi::StartResult result;
@@ -1314,7 +1319,7 @@ Rectangle CameraData::scaleIspCrop(const Rectangle &ispCrop) const
 	return nativeCrop;
 }
 
-void CameraData::applyScalerCrop(const ControlList &controls)
+void CameraData::applyScalerCropAndWindows(const ControlList &controls)
 {
 	const auto &scalerCropRPi = controls.get<Span<const Rectangle>>(controls::rpi::ScalerCrops);
 	const auto &scalerCropCore = controls.get<Rectangle>(controls::ScalerCrop);
@@ -1361,7 +1366,32 @@ void CameraData::applyScalerCrop(const ControlList &controls)
 
 		if (ispCrop != cropParams_.at(i).ispCrop) {
 			cropParams_.at(i).ispCrop = ispCrop;
-			platformSetIspCrop(cropParams_.at(i).ispIndex, ispCrop);
+			platformSetIspCrop(ispIndices_[i], ispCrop);
+		}
+	}
+
+	const auto &outputWindows = controls.get<Span<const Rectangle>>(controls::rpi::OutputWindows);
+	if (outputWindows) {
+		for (unsigned int i = 0; i < outputWindows->size() && i < outputWindows_.size(); i++) {
+			const Rectangle &outputWindow = outputWindows->data()[i];
+			const StreamConfiguration &streamConfig = streams_[i]->configuration();
+
+			/* Do some basic checks here; platformSetOutputWindow will do the rest. */
+			if (outputWindow.x + outputWindow.width > streamConfig.size.width)
+				LOG(RPI, Warning) << "Offset " << outputWindow.x << " plus width "
+						  << outputWindow.width << " exceed maximum "
+						  << streamConfig.size.width;
+			else if (outputWindow.y + outputWindow.height > streamConfig.size.height)
+				LOG(RPI, Warning) << "Offset " << outputWindow.y << " plus height "
+						  << outputWindow.height << " exceed maximum "
+						  << streamConfig.size.height;
+			else {
+				LOG(RPI, Debug) << "Ouptut window " << i << " set to "
+						<< outputWindow.width << " x " << outputWindow.height;
+				/* platformSetOutputWindow will issue a warning if it fails. */
+				if (platformSetOutputWindow(i, outputWindow) == 0)
+					outputWindows_[i] = outputWindow;
+			}
 		}
 	}
 }
@@ -1530,6 +1560,11 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request
 									    crops.size()));
 		}
 	}
+
+	if (outputWindows_.size())
+		request->_d()->metadata().set(controls::rpi::OutputWindows,
+					      Span<const Rectangle>(outputWindows_.data(),
+								    outputWindows_.size()));
 }
 
 static bool isControlDelayed(unsigned int id)

src/libcamera/pipeline/rpi/common/pipeline_base.h

@@ -82,8 +82,9 @@ class CameraData : public Camera::Private
 	void setSensorControls(ControlList &controls);
 
 	Rectangle scaleIspCrop(const Rectangle &ispCrop) const;
-	void applyScalerCrop(const ControlList &controls);
-	virtual void platformSetIspCrop(unsigned int index, const Rectangle &ispCrop) = 0;
+	void applyScalerCropAndWindows(const ControlList &controls);
+	virtual void platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop) = 0;
+	virtual int platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow) = 0;
 
 	void cameraTimeout();
 	void frameStarted(uint32_t sequence);
@@ -135,22 +136,29 @@ class CameraData : public Camera::Private
 	IPACameraSensorInfo sensorInfo_;
 
 	struct CropParams {
-		CropParams(Rectangle ispCrop_, Size ispMinCropSize_, unsigned int ispIndex_)
-			: ispCrop(ispCrop_), ispMinCropSize(ispMinCropSize_), ispIndex(ispIndex_)
+		CropParams(Rectangle ispCrop_, Size ispMinCropSize_)
+			: ispCrop(ispCrop_), ispMinCropSize(ispMinCropSize_)
 		{
 		}
 
 		/* Crop in ISP (camera mode) pixels */
 		Rectangle ispCrop;
 		/* Minimum crop size in ISP output pixels */
 		Size ispMinCropSize;
-		/* Index of the ISP output channel for this crop */
-		unsigned int ispIndex;
 	};
 
+	/*
+	 * Vector recording the ISP index (output branch number) for the streams, ordered
+	 * according to the stream's position in the CameraConfiguration.
+	 */
+	std::vector<unsigned int> ispIndices_;
+
 	/* Mapping of CropParams keyed by the output stream order in CameraConfiguration */
 	std::map<unsigned int, CropParams> cropParams_;
 
+	/* Vector of output windows, again ordered according to the CameraConfiguration. */
+	std::vector<Rectangle> outputWindows_;
+
 	unsigned int startupFrameCount_;
 	unsigned int invalidFrameCount_;
 

src/libcamera/pipeline/rpi/pisp/pisp.cpp

@@ -51,6 +51,11 @@ enum class Cfe : unsigned int { Output0, Embedded, Stats, Config };
 enum class Isp : unsigned int { Input, Output0, Output1, TdnInput, TdnOutput,
 				StitchInput, StitchOutput, Config };
 
+struct OutputInfo {
+	unsigned int minWidth;
+	unsigned int alignment;
+};
+
 /* Offset for all compressed buffers; mode for TDN and Stitch. */
 constexpr unsigned int DefaultCompressionOffset = 2048;
 constexpr unsigned int DefaultCompressionMode = 1;
@@ -691,12 +696,15 @@ unsigned int getFormatAlignment(const V4L2PixelFormat &fourcc)
 
 /* Calculate the amount of software downscale required (which is a power of 2). */
 unsigned int calculateSwDownscale(const V4L2DeviceFormat &format, unsigned int largestWidth,
-				  unsigned int platformMaxDownscale)
+				  unsigned int platformMaxDownscale, OutputInfo *outputInfo = nullptr)
 {
 	unsigned int formatAlignment = getFormatAlignment(format.fourcc);
 	unsigned int maxDownscale = platformMaxDownscale * 16 / formatAlignment;
 	unsigned int limitWidth = largestWidth / maxDownscale;
 
+	if (outputInfo)
+		*outputInfo = { limitWidth, formatAlignment };
+
 	unsigned int hwWidth = format.size.width;
 	unsigned int swDownscale = 1;
 	for (; hwWidth < limitWidth; hwWidth *= 2, swDownscale *= 2);
@@ -814,7 +822,8 @@ class PiSPCameraData final : public RPi::CameraData
 	bool calculateCscConfiguration(const V4L2DeviceFormat &v4l2Format, pisp_be_ccm_config &csc);
 	int configureBe(const std::optional<ColorSpace> &yuvColorSpace);
 
-	void platformSetIspCrop(unsigned int index, const Rectangle &ispCrop) override;
+	void platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop) override;
+	int platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow) override;
 
 	void prepareCfe();
 	void prepareBe(uint32_t bufferId, bool stitchSwapBuffers);
@@ -842,6 +851,8 @@ class PiSPCameraData final : public RPi::CameraData
 	}
 
 	std::string last_dump_file_;
+
+	std::vector<OutputInfo> outputInfo_;
 };
 
 class PipelineHandlerPiSP : public RPi::PipelineHandlerBase
@@ -1505,7 +1516,10 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf
 							      s == &isp_[Isp::Output1]; }),
 		       streams_.end());
 
+	ispIndices_.clear();
 	cropParams_.clear();
+	outputWindows_.clear();
+
 	for (unsigned int i = 0; i < outStreams.size(); i++) {
 		StreamConfiguration *cfg = outStreams[i].cfg;
 		unsigned int ispIndex;
@@ -1524,6 +1538,7 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf
 			beEnables |= PISP_BE_RGB_ENABLE_OUTPUT0;
 			ispIndex = 0;
 		}
+		ispIndices_.push_back(ispIndex);
 
 		format = outStreams[i].format;
 		bool needs32BitConversion = adjustDeviceFormat(format);
@@ -1535,8 +1550,10 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf
 		PixelFormat pixFmt = format.fourcc.toPixelFormat();
 
 		/* If there's excessive downscaling we'll do some of it in software. */
+		outputInfo_.emplace_back(OutputInfo{ 0, 0 });
 		unsigned int swDownscale = calculateSwDownscale(format, largestWidth,
-								be_->GetMaxDownscale());
+								be_->GetMaxDownscale(),
+								&outputInfo_.back());
 		unsigned int hwWidth = format.size.width * swDownscale;
 		format.size.width = hwWidth;
 
@@ -1566,6 +1583,8 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf
 		if (needs32BitConversion)
 			flags |= StreamFlag::Needs32bitConv;
 
+		outputWindows_.push_back(Rectangle(outStreams[i].cfg->size));
+
 		/* Set smallest selection the ISP will allow. */
 		Size minCrop{ 32, 32 };
 
@@ -1590,7 +1609,7 @@ int PiSPCameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConf
 		 */
 		cropParams_.emplace(std::piecewise_construct,
 				    std::forward_as_tuple(outStreams[i].index),
-				    std::forward_as_tuple(ispCrop, minCrop, ispIndex));
+				    std::forward_as_tuple(ispCrop, minCrop));
 
 		cfg->setStream(stream);
 		stream->setFlags(flags);
@@ -2138,7 +2157,7 @@ int PiSPCameraData::configureBe(const std::optional<ColorSpace> &yuvColorSpace)
 	return 0;
 }
 
-void PiSPCameraData::platformSetIspCrop(unsigned int index, const Rectangle &ispCrop)
+void PiSPCameraData::platformSetIspCrop(unsigned int ispIndex, const Rectangle &ispCrop)
 {
 	pisp_be_crop_config beCrop = {
 		static_cast<uint16_t>(ispCrop.x),
@@ -2147,8 +2166,55 @@ void PiSPCameraData::platformSetIspCrop(unsigned int index, const Rectangle &isp
 		static_cast<uint16_t>(ispCrop.height)
 	};
 
-	LOG(RPI, Debug) << "Output " << index << " " << ispCrop.toString();
-	be_->SetCrop(index, beCrop);
+	LOG(RPI, Debug) << "Output " << ispIndex << " " << ispCrop.toString();
+	be_->SetCrop(ispIndex, beCrop);
+}
+
+int PiSPCameraData::platformSetOutputWindow(unsigned int streamIndex, const Rectangle &outputWindow)
+{
+	pisp_be_output_format_config config;
+	pisp_be_output_format_extra extra;
+	BackEnd::SmartResize resize;
+	const OutputInfo &outputInfo = outputInfo_[streamIndex];
+	unsigned int ispIndex = ispIndices_[streamIndex];
+
+	be_->GetOutputFormat(ispIndex, config, extra);
+
+	/*
+	 * We insist on a minimum size that ensures no software downscaling is ever required,
+	 * which limits the amount of reconfiguration one might have to do here.
+	 */
+	if (outputWindow.width < outputInfo.minWidth) {
+		LOG(RPI, Warning) << "Window width " << outputWindow.width
+				  << " less than minimum value " << outputInfo.minWidth;
+		return -1;
+	} else if (outputWindow.height < 2) {
+		LOG(RPI, Warning) << "Window height " << outputWindow.height
+				  << " less than minimum value 2";
+		return -1;
+	} else if (outputWindow.x % outputInfo.alignment) {
+		LOG(RPI, Warning) << "Window x offset " << outputWindow.x
+				  << " does not match pixel alignment " << outputInfo.alignment;
+		return -1;
+	} else if (outputWindow.y & 1) {
+		LOG(RPI, Warning) << "Window y offset " << outputWindow.y
+				  << " must be even";
+		return -1;
+	}
+
+	resize.width = outputWindow.width;
+	resize.height = outputWindow.height;
+	be_->SetSmartResize(ispIndex, resize);
+
+	config.image.width = outputWindow.width;
+	config.image.height = outputWindow.height;
+
+	extra.offset_x = outputWindow.x;
+	extra.offset_y = outputWindow.y;
+
+	be_->SetOutputFormat(ispIndex, config, extra);
+
+	return 0;
 }
 
 int PiSPCameraData::platformInitIpa(ipa::RPi::InitParams &params)
@@ -2307,7 +2373,7 @@ void PiSPCameraData::tryRunPipeline()
 	ASSERT(request->metadata().empty());
 
 	/* See if a new ScalerCrop value needs to be applied. */
-	applyScalerCrop(request->controls());
+	applyScalerCropAndWindows(request->controls());
 
 	fillRequestMetadata(job.sensorControls, request);