Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ services:
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "select" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "multiselect" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "input" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "calculatedValue" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "country" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "countrymultiselect" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "language" }
Expand All @@ -24,6 +23,11 @@ services:
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "email" }
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "gender" }

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter\CalculatedValueAdapter:
shared: false
tags:
- { name: "pimcore.generic_data_index.data-object.search_index_field_definition", type: "calculatedValue" }

Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter\NumericAdapter:
shared: false
tags:
Expand Down
12 changes: 12 additions & 0 deletions doc/01_Installation/02_Upgrade.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,18 @@ description: Version-specific upgrade instructions and breaking changes for the

# Upgrade Information

## Upgrade to 2026.2.9

### Re-indexing required

- [Indexing] `CalculatedValue` fields are no longer always mapped as text/keyword: fields configured with
element type `boolean` are now mapped as `boolean`, `numeric` as `double` and `date` as `date`
(`strict_date_time_no_millis`). The text-based element types (`input`, `textarea`, `html` — including the
default) keep the existing text/keyword mapping. This makes term filters, aggregations and the Studio grid
filters work on typed calculated fields.
- The mapping change is applied by `bin/console generic-data-index:update:index`, which recreates the
affected data object indices and queues all elements for re-indexing.

## Upgrade to 2026.2.8

### Re-indexing required
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
<?php
declare(strict_types=1);

/**
* This source file is available under the terms of the
* Pimcore Open Core License (POCL)
* Full copyright and license information is available in
* LICENSE.md which is distributed with this source code.
*
* @copyright Copyright (c) Pimcore GmbH (https://www.pimcore.com)
* @license Pimcore Open Core License (POCL)
*/

namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter;

use Carbon\Carbon;
use Carbon\Exceptions\InvalidFormatException;
use DateTimeInterface;
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\DefaultSearch\AttributeType;
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\FieldDefinitionServiceInterface;
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface;
use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface;
use Pimcore\Model\DataObject\ClassDefinition\Data\CalculatedValue;

/**
* Maps calculated value fields based on their configured element type instead of
* always falling back to a text mapping, so that boolean/numeric/date calculated
* values stay filterable and aggregatable in the search index.
*
* @internal
*/
final class CalculatedValueAdapter extends AbstractAdapter
{
use NumericMappingTrait;

private const ELEMENT_TYPE_BOOLEAN = 'boolean';

private const ELEMENT_TYPE_NUMERIC = 'numeric';

private const ELEMENT_TYPE_DATE = 'date';

public function __construct(
protected SearchIndexConfigServiceInterface $searchIndexConfigService,
protected FieldDefinitionServiceInterface $fieldDefinitionService,
private readonly IndexMappingServiceInterface $indexMappingService,
) {
parent::__construct(
$searchIndexConfigService,
$fieldDefinitionService
);
}

public function getIndexMapping(): array
{
return match ($this->getElementType()) {
self::ELEMENT_TYPE_BOOLEAN => [
'type' => AttributeType::BOOLEAN->value,
],
// no integer flag on calculated fields, so always 64-bit double
self::ELEMENT_TYPE_NUMERIC => $this->getNumericMapping(integer: false),
Comment thread
kingjia90 marked this conversation as resolved.
self::ELEMENT_TYPE_DATE => [
'type' => AttributeType::DATE->value,
'format' => 'strict_date_time_no_millis',
],
default => $this->indexMappingService->getMappingForTextKeyword(
$this->searchIndexConfigService->getSearchAnalyzerAttributes()
),
};
}

public function normalize(mixed $value): mixed
{
return match ($this->getElementType()) {
self::ELEMENT_TYPE_BOOLEAN => $this->normalizeBoolean($value),
self::ELEMENT_TYPE_NUMERIC => is_numeric($value) ? (float) $value : null,
self::ELEMENT_TYPE_DATE => $this->normalizeDate($value),
default => $this->normalizeText($value),
};
}

private function normalizeDate(mixed $value): ?string
{
if ($value instanceof DateTimeInterface) {
return $value->format(DateTimeInterface::ATOM);
}

// Class calculators are typed to return strings and the query store only
// yields strings, so date objects rarely reach this point; unparseable
// values degrade to null instead of failing the whole document.
if (is_string($value) && trim($value) !== '') {
try {
return (new Carbon($value))->format(DateTimeInterface::ATOM);
} catch (InvalidFormatException) {
return null;
}
}

return null;
}

private function normalizeBoolean(mixed $value): ?bool
{
if ($value === null || is_bool($value)) {
return $value;
}

// Calculators may return loosely typed values; only recognized boolean
// representations are indexed, anything else becomes null instead of
// silently reversing filters (e.g. 'false' must not turn into true).
return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE);
}

private function normalizeText(mixed $value): mixed
{
if (is_string($value) && $value !== '') {
return preg_replace("/src=(['\"])data:[^;]+;base64,.+?\\1/", '', $value);
}

return parent::normalize($value);
}

private function getElementType(): ?string
{
$fieldDefinition = $this->getFieldDefinition();
if (!$fieldDefinition instanceof CalculatedValue) {
return null;
}

return $fieldDefinition->getElementType();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
<?php
declare(strict_types=1);

/**
* This source file is available under the terms of the
* Pimcore Open Core License (POCL)
* Full copyright and license information is available in
* LICENSE.md which is distributed with this source code.
*
* @copyright Copyright (c) Pimcore GmbH (https://www.pimcore.com)
* @license Pimcore Open Core License (POCL)
*/

namespace Pimcore\Bundle\GenericDataIndexBundle\Tests\Unit\SearchIndexAdapter\DataObject\FieldDefinitionAdapter;

use Carbon\Carbon;
use Codeception\Test\Unit;
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\FieldDefinitionServiceInterface;
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter\CalculatedValueAdapter;
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface;
use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface;
use Pimcore\Model\DataObject\ClassDefinition\Data\CalculatedValue;

/**
* @internal
*/
final class CalculatedValueAdapterTest extends Unit
{
public function testGetSearchIndexMappingForBooleanElementType(): void
{
$adapter = $this->createAdapter('boolean');

$this->assertSame([
'type' => 'boolean',
], $adapter->getIndexMapping());
}

public function testGetSearchIndexMappingForNumericElementType(): void
{
$adapter = $this->createAdapter('numeric');

$this->assertSame([
'type' => 'double',
], $adapter->getIndexMapping());
}

public function testGetSearchIndexMappingForDateElementType(): void
{
$adapter = $this->createAdapter('date');

$this->assertSame([
'type' => 'date',
'format' => 'strict_date_time_no_millis',
], $adapter->getIndexMapping());
}

public function testGetSearchIndexMappingFallsBackToTextKeyword(): void
{
$textKeywordMapping = [
'type' => 'text',
'fields' => [
'keyword' => [
'type' => 'keyword',
'ignore_above' => 1024,
],
],
];

foreach (['input', 'textarea', 'html'] as $elementType) {
$adapter = $this->createAdapter($elementType, $textKeywordMapping);

$this->assertSame($textKeywordMapping, $adapter->getIndexMapping());
}
}

/**
* Calculators are free to return loosely typed values (e.g. '0'/'1' from
* expression results), which a boolean index mapping would reject.
*/
public function testNormalizeCastsBooleanElementTypeValues(): void
{
$adapter = $this->createAdapter('boolean');

$this->assertTrue($adapter->normalize(true));
$this->assertFalse($adapter->normalize(false));
$this->assertTrue($adapter->normalize('1'));
$this->assertFalse($adapter->normalize('0'));
$this->assertTrue($adapter->normalize(1));
$this->assertFalse($adapter->normalize(0));
$this->assertTrue($adapter->normalize('true'));
$this->assertFalse($adapter->normalize('false'));
$this->assertTrue($adapter->normalize('yes'));
$this->assertFalse($adapter->normalize('no'));
$this->assertNull($adapter->normalize('not a boolean'));
$this->assertNull($adapter->normalize(null));
}

public function testNormalizeCastsNumericElementTypeValues(): void
{
$adapter = $this->createAdapter('numeric');

$this->assertSame(1.5, $adapter->normalize(1.5));
$this->assertSame(3.0, $adapter->normalize(3));
$this->assertSame(2.25, $adapter->normalize('2.25'));
$this->assertNull($adapter->normalize('not a number'));
$this->assertNull($adapter->normalize(null));
}

public function testNormalizeFormatsDateElementTypeValues(): void
{
$adapter = $this->createAdapter('date');

$this->assertSame(
'2024-06-15T10:30:00+00:00',
$adapter->normalize(Carbon::create(2024, 6, 15, 10, 30, 0, 'UTC'))
);
$this->assertSame(
'2024-06-15T10:30:00+02:00',
$adapter->normalize('2024-06-15T10:30:00+02:00')
);
$this->assertNull($adapter->normalize('not a date'));
$this->assertNull($adapter->normalize(''));
$this->assertNull($adapter->normalize(null));
}

/**
* Class calculators are typed to return strings and the query store casts every
* value to string, so date strings without timezone information (e.g. a stored
* Carbon "Y-m-d H:i:s" representation) are the common case and must be indexed.
*/
public function testNormalizeParsesDateStringsWithoutTimezone(): void
{
$adapter = $this->createAdapter('date');

$this->assertStringStartsWith('2024-06-15T10:30:00', $adapter->normalize('2024-06-15 10:30:00'));
$this->assertStringStartsWith('2024-06-15T00:00:00', $adapter->normalize('2024-06-15'));
}

public function testNormalizeKeepsTextElementTypeBehavior(): void
{
$adapter = $this->createAdapter('html');

$this->assertSame(
'<img alt="test">',
$adapter->normalize('<img src="data:image/png;base64,iVBORw0KGgo=" alt="test">')
);
$this->assertSame('plain text', $adapter->normalize('plain text'));
}

private function createAdapter(string $elementType, array $textKeywordMapping = []): CalculatedValueAdapter
{
$indexMappingServiceMock = $this->makeEmpty(IndexMappingServiceInterface::class, [
'getMappingForTextKeyword' => $textKeywordMapping,
]);

$adapter = new CalculatedValueAdapter(
$this->makeEmpty(SearchIndexConfigServiceInterface::class),
$this->makeEmpty(FieldDefinitionServiceInterface::class),
$indexMappingServiceMock
);

$fieldDefinition = new CalculatedValue();
$fieldDefinition->setElementType($elementType);
$adapter->setFieldDefinition($fieldDefinition);

return $adapter;
}
}
Loading