-
Notifications
You must be signed in to change notification settings - Fork 20
Map calculated value fields according to their configured element type #488
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
kingjia90
merged 4 commits into
pimcore:2026.2
from
andreaswagner-rnab:fix/calculated-value-element-type-mapping
Sep 16, 2026
+316
−1
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
b8d8f42
Map calculated value fields according to their configured element type
andreaswagner-rnab abd7cc9
Validate recognized boolean representations instead of casting
andreaswagner-rnab 92f487e
Merge branch '2026.2' into fix/calculated-value-element-type-mapping
kingjia90 26301b8
Map calculated numeric fields as double and parse date strings
kingjia90 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
131 changes: 131 additions & 0 deletions
131
...chIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/CalculatedValueAdapter.php
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| <?php | ||
| declare(strict_types=1); | ||
|
|
||
| /** | ||
| * This source file is available under the terms of the | ||
| * Pimcore Open Core License (POCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (https://www.pimcore.com) | ||
| * @license Pimcore Open Core License (POCL) | ||
| */ | ||
|
|
||
| namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter; | ||
|
|
||
| use Carbon\Carbon; | ||
| use Carbon\Exceptions\InvalidFormatException; | ||
| use DateTimeInterface; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\DefaultSearch\AttributeType; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\FieldDefinitionServiceInterface; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface; | ||
| use Pimcore\Model\DataObject\ClassDefinition\Data\CalculatedValue; | ||
|
|
||
| /** | ||
| * Maps calculated value fields based on their configured element type instead of | ||
| * always falling back to a text mapping, so that boolean/numeric/date calculated | ||
| * values stay filterable and aggregatable in the search index. | ||
| * | ||
| * @internal | ||
| */ | ||
| final class CalculatedValueAdapter extends AbstractAdapter | ||
| { | ||
| use NumericMappingTrait; | ||
|
|
||
| private const ELEMENT_TYPE_BOOLEAN = 'boolean'; | ||
|
|
||
| private const ELEMENT_TYPE_NUMERIC = 'numeric'; | ||
|
|
||
| private const ELEMENT_TYPE_DATE = 'date'; | ||
|
|
||
| public function __construct( | ||
| protected SearchIndexConfigServiceInterface $searchIndexConfigService, | ||
| protected FieldDefinitionServiceInterface $fieldDefinitionService, | ||
| private readonly IndexMappingServiceInterface $indexMappingService, | ||
| ) { | ||
| parent::__construct( | ||
| $searchIndexConfigService, | ||
| $fieldDefinitionService | ||
| ); | ||
| } | ||
|
|
||
| public function getIndexMapping(): array | ||
| { | ||
| return match ($this->getElementType()) { | ||
| self::ELEMENT_TYPE_BOOLEAN => [ | ||
| 'type' => AttributeType::BOOLEAN->value, | ||
| ], | ||
| // no integer flag on calculated fields, so always 64-bit double | ||
| self::ELEMENT_TYPE_NUMERIC => $this->getNumericMapping(integer: false), | ||
| self::ELEMENT_TYPE_DATE => [ | ||
| 'type' => AttributeType::DATE->value, | ||
| 'format' => 'strict_date_time_no_millis', | ||
| ], | ||
| default => $this->indexMappingService->getMappingForTextKeyword( | ||
| $this->searchIndexConfigService->getSearchAnalyzerAttributes() | ||
| ), | ||
| }; | ||
| } | ||
|
|
||
| public function normalize(mixed $value): mixed | ||
| { | ||
| return match ($this->getElementType()) { | ||
| self::ELEMENT_TYPE_BOOLEAN => $this->normalizeBoolean($value), | ||
| self::ELEMENT_TYPE_NUMERIC => is_numeric($value) ? (float) $value : null, | ||
| self::ELEMENT_TYPE_DATE => $this->normalizeDate($value), | ||
| default => $this->normalizeText($value), | ||
| }; | ||
| } | ||
|
|
||
| private function normalizeDate(mixed $value): ?string | ||
| { | ||
| if ($value instanceof DateTimeInterface) { | ||
| return $value->format(DateTimeInterface::ATOM); | ||
| } | ||
|
|
||
| // Class calculators are typed to return strings and the query store only | ||
| // yields strings, so date objects rarely reach this point; unparseable | ||
| // values degrade to null instead of failing the whole document. | ||
| if (is_string($value) && trim($value) !== '') { | ||
| try { | ||
| return (new Carbon($value))->format(DateTimeInterface::ATOM); | ||
| } catch (InvalidFormatException) { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| return null; | ||
| } | ||
|
|
||
| private function normalizeBoolean(mixed $value): ?bool | ||
| { | ||
| if ($value === null || is_bool($value)) { | ||
| return $value; | ||
| } | ||
|
|
||
| // Calculators may return loosely typed values; only recognized boolean | ||
| // representations are indexed, anything else becomes null instead of | ||
| // silently reversing filters (e.g. 'false' must not turn into true). | ||
| return filter_var($value, FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE); | ||
| } | ||
|
|
||
| private function normalizeText(mixed $value): mixed | ||
| { | ||
| if (is_string($value) && $value !== '') { | ||
| return preg_replace("/src=(['\"])data:[^;]+;base64,.+?\\1/", '', $value); | ||
| } | ||
|
|
||
| return parent::normalize($value); | ||
| } | ||
|
|
||
| private function getElementType(): ?string | ||
| { | ||
| $fieldDefinition = $this->getFieldDefinition(); | ||
| if (!$fieldDefinition instanceof CalculatedValue) { | ||
| return null; | ||
| } | ||
|
|
||
| return $fieldDefinition->getElementType(); | ||
| } | ||
| } | ||
168 changes: 168 additions & 0 deletions
168
.../Unit/SearchIndexAdapter/DataObject/FieldDefinitionAdapter/CalculatedValueAdapterTest.php
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| <?php | ||
| declare(strict_types=1); | ||
|
|
||
| /** | ||
| * This source file is available under the terms of the | ||
| * Pimcore Open Core License (POCL) | ||
| * Full copyright and license information is available in | ||
| * LICENSE.md which is distributed with this source code. | ||
| * | ||
| * @copyright Copyright (c) Pimcore GmbH (https://www.pimcore.com) | ||
| * @license Pimcore Open Core License (POCL) | ||
| */ | ||
|
|
||
| namespace Pimcore\Bundle\GenericDataIndexBundle\Tests\Unit\SearchIndexAdapter\DataObject\FieldDefinitionAdapter; | ||
|
|
||
| use Carbon\Carbon; | ||
| use Codeception\Test\Unit; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DataObject\FieldDefinitionServiceInterface; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\DataObject\FieldDefinitionAdapter\CalculatedValueAdapter; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\IndexMappingServiceInterface; | ||
| use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface; | ||
| use Pimcore\Model\DataObject\ClassDefinition\Data\CalculatedValue; | ||
|
|
||
| /** | ||
| * @internal | ||
| */ | ||
| final class CalculatedValueAdapterTest extends Unit | ||
| { | ||
| public function testGetSearchIndexMappingForBooleanElementType(): void | ||
| { | ||
| $adapter = $this->createAdapter('boolean'); | ||
|
|
||
| $this->assertSame([ | ||
| 'type' => 'boolean', | ||
| ], $adapter->getIndexMapping()); | ||
| } | ||
|
|
||
| public function testGetSearchIndexMappingForNumericElementType(): void | ||
| { | ||
| $adapter = $this->createAdapter('numeric'); | ||
|
|
||
| $this->assertSame([ | ||
| 'type' => 'double', | ||
| ], $adapter->getIndexMapping()); | ||
| } | ||
|
|
||
| public function testGetSearchIndexMappingForDateElementType(): void | ||
| { | ||
| $adapter = $this->createAdapter('date'); | ||
|
|
||
| $this->assertSame([ | ||
| 'type' => 'date', | ||
| 'format' => 'strict_date_time_no_millis', | ||
| ], $adapter->getIndexMapping()); | ||
| } | ||
|
|
||
| public function testGetSearchIndexMappingFallsBackToTextKeyword(): void | ||
| { | ||
| $textKeywordMapping = [ | ||
| 'type' => 'text', | ||
| 'fields' => [ | ||
| 'keyword' => [ | ||
| 'type' => 'keyword', | ||
| 'ignore_above' => 1024, | ||
| ], | ||
| ], | ||
| ]; | ||
|
|
||
| foreach (['input', 'textarea', 'html'] as $elementType) { | ||
| $adapter = $this->createAdapter($elementType, $textKeywordMapping); | ||
|
|
||
| $this->assertSame($textKeywordMapping, $adapter->getIndexMapping()); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Calculators are free to return loosely typed values (e.g. '0'/'1' from | ||
| * expression results), which a boolean index mapping would reject. | ||
| */ | ||
| public function testNormalizeCastsBooleanElementTypeValues(): void | ||
| { | ||
| $adapter = $this->createAdapter('boolean'); | ||
|
|
||
| $this->assertTrue($adapter->normalize(true)); | ||
| $this->assertFalse($adapter->normalize(false)); | ||
| $this->assertTrue($adapter->normalize('1')); | ||
| $this->assertFalse($adapter->normalize('0')); | ||
| $this->assertTrue($adapter->normalize(1)); | ||
| $this->assertFalse($adapter->normalize(0)); | ||
| $this->assertTrue($adapter->normalize('true')); | ||
| $this->assertFalse($adapter->normalize('false')); | ||
| $this->assertTrue($adapter->normalize('yes')); | ||
| $this->assertFalse($adapter->normalize('no')); | ||
| $this->assertNull($adapter->normalize('not a boolean')); | ||
| $this->assertNull($adapter->normalize(null)); | ||
| } | ||
|
|
||
| public function testNormalizeCastsNumericElementTypeValues(): void | ||
| { | ||
| $adapter = $this->createAdapter('numeric'); | ||
|
|
||
| $this->assertSame(1.5, $adapter->normalize(1.5)); | ||
| $this->assertSame(3.0, $adapter->normalize(3)); | ||
| $this->assertSame(2.25, $adapter->normalize('2.25')); | ||
| $this->assertNull($adapter->normalize('not a number')); | ||
| $this->assertNull($adapter->normalize(null)); | ||
| } | ||
|
|
||
| public function testNormalizeFormatsDateElementTypeValues(): void | ||
| { | ||
| $adapter = $this->createAdapter('date'); | ||
|
|
||
| $this->assertSame( | ||
| '2024-06-15T10:30:00+00:00', | ||
| $adapter->normalize(Carbon::create(2024, 6, 15, 10, 30, 0, 'UTC')) | ||
| ); | ||
| $this->assertSame( | ||
| '2024-06-15T10:30:00+02:00', | ||
| $adapter->normalize('2024-06-15T10:30:00+02:00') | ||
| ); | ||
| $this->assertNull($adapter->normalize('not a date')); | ||
| $this->assertNull($adapter->normalize('')); | ||
| $this->assertNull($adapter->normalize(null)); | ||
| } | ||
|
|
||
| /** | ||
| * Class calculators are typed to return strings and the query store casts every | ||
| * value to string, so date strings without timezone information (e.g. a stored | ||
| * Carbon "Y-m-d H:i:s" representation) are the common case and must be indexed. | ||
| */ | ||
| public function testNormalizeParsesDateStringsWithoutTimezone(): void | ||
| { | ||
| $adapter = $this->createAdapter('date'); | ||
|
|
||
| $this->assertStringStartsWith('2024-06-15T10:30:00', $adapter->normalize('2024-06-15 10:30:00')); | ||
| $this->assertStringStartsWith('2024-06-15T00:00:00', $adapter->normalize('2024-06-15')); | ||
| } | ||
|
|
||
| public function testNormalizeKeepsTextElementTypeBehavior(): void | ||
| { | ||
| $adapter = $this->createAdapter('html'); | ||
|
|
||
| $this->assertSame( | ||
| '<img alt="test">', | ||
| $adapter->normalize('<img src="data:image/png;base64,iVBORw0KGgo=" alt="test">') | ||
| ); | ||
| $this->assertSame('plain text', $adapter->normalize('plain text')); | ||
| } | ||
|
|
||
| private function createAdapter(string $elementType, array $textKeywordMapping = []): CalculatedValueAdapter | ||
| { | ||
| $indexMappingServiceMock = $this->makeEmpty(IndexMappingServiceInterface::class, [ | ||
| 'getMappingForTextKeyword' => $textKeywordMapping, | ||
| ]); | ||
|
|
||
| $adapter = new CalculatedValueAdapter( | ||
| $this->makeEmpty(SearchIndexConfigServiceInterface::class), | ||
| $this->makeEmpty(FieldDefinitionServiceInterface::class), | ||
| $indexMappingServiceMock | ||
| ); | ||
|
|
||
| $fieldDefinition = new CalculatedValue(); | ||
| $fieldDefinition->setElementType($elementType); | ||
| $adapter->setFieldDefinition($fieldDefinition); | ||
|
|
||
| return $adapter; | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.