From 0b265733142a21de91ed90d0e197c48b0561bb1e Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Mon, 27 Oct 2025 17:23:15 +0000 Subject: [PATCH] Search: Started work to make search result size consistent --- app/Entities/Models/Entity.php | 13 ++ app/Entities/Models/EntityTable.php | 37 +++++ app/Entities/Queries/EntityQueries.php | 26 +++- app/Entities/Tools/EntityHydrator.php | 151 +++++++++++++++++++ app/Entities/Tools/MixedEntityListLoader.php | 2 +- app/Search/SearchRunner.php | 134 +++++++--------- 6 files changed, 284 insertions(+), 79 deletions(-) create mode 100644 app/Entities/Models/EntityTable.php create mode 100644 app/Entities/Tools/EntityHydrator.php diff --git a/app/Entities/Models/Entity.php b/app/Entities/Models/Entity.php index 77393cbbc..b47a029ec 100644 --- a/app/Entities/Models/Entity.php +++ b/app/Entities/Models/Entity.php @@ -471,4 +471,17 @@ abstract class Entity extends Model implements return $contentFields; } + + /** + * Create a new instance for the given entity type. + */ + public static function instanceFromType(string $type): self + { + return match ($type) { + 'page' => new Page(), + 'chapter' => new Chapter(), + 'book' => new Book(), + 'bookshelf' => new Bookshelf(), + }; + } } diff --git a/app/Entities/Models/EntityTable.php b/app/Entities/Models/EntityTable.php new file mode 100644 index 000000000..50112a8e9 --- /dev/null +++ b/app/Entities/Models/EntityTable.php @@ -0,0 +1,37 @@ +make(PermissionApplicator::class)->restrictEntityQuery($query); + } + + /** + * Get the entity jointPermissions this is connected to. + */ + public function jointPermissions(): HasMany + { + return $this->hasMany(JointPermission::class, 'entity_id')->whereColumn('entity_type', '=', 'entities.type'); + } +} diff --git a/app/Entities/Queries/EntityQueries.php b/app/Entities/Queries/EntityQueries.php index a7a037916..c27cc61cc 100644 --- a/app/Entities/Queries/EntityQueries.php +++ b/app/Entities/Queries/EntityQueries.php @@ -3,7 +3,10 @@ namespace BookStack\Entities\Queries; use BookStack\Entities\Models\Entity; +use BookStack\Entities\Models\EntityTable; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Query\JoinClause; +use Illuminate\Support\Facades\DB; use InvalidArgumentException; class EntityQueries @@ -32,12 +35,31 @@ class EntityQueries return $queries->findVisibleById($entityId); } + /** + * Start a query across all entity types. + * Combines the description/text fields into a single 'description' field. + * @return Builder + */ + public function visibleForList(): Builder + { + $rawDescriptionField = DB::raw('COALESCE(description, text) as description'); + return EntityTable::query()->scopes('visible') + ->select(['id', 'type', 'name', 'slug', 'book_id', 'chapter_id', 'created_at', 'updated_at', 'draft', $rawDescriptionField]) + ->leftJoin('entity_container_data', function (JoinClause $join) { + $join->on('entity_container_data.entity_id', '=', 'entities.id') + ->on('entity_container_data.entity_type', '=', 'entities.type'); + })->leftJoin('entity_page_data', function (JoinClause $join) { + $join->on('entity_page_data.page_id', '=', 'entities.id') + ->where('entities.type', '=', 'page'); + }); + } + /** * Start a query of visible entities of the given type, * suitable for listing display. * @return Builder */ - public function visibleForList(string $entityType): Builder + public function visibleForListForType(string $entityType): Builder { $queries = $this->getQueriesForType($entityType); return $queries->visibleForList(); @@ -48,7 +70,7 @@ class EntityQueries * suitable for using the contents of the items. * @return Builder */ - public function visibleForContent(string $entityType): Builder + public function visibleForContentForType(string $entityType): Builder { $queries = $this->getQueriesForType($entityType); return $queries->visibleForContent(); diff --git a/app/Entities/Tools/EntityHydrator.php b/app/Entities/Tools/EntityHydrator.php new file mode 100644 index 000000000..fbfc8762e --- /dev/null +++ b/app/Entities/Tools/EntityHydrator.php @@ -0,0 +1,151 @@ +entities = $entities; + $this->loadTags = $loadTags; + $this->loadParents = $loadParents; + } + + /** + * Hydrate the entities of this hydrator to return a list of entities represented + * in their original intended models. + * @return Entity[] + */ + public function hydrate(): array + { + $hydrated = []; + + foreach ($this->entities as $entity) { + $data = $entity->toArray(); + $instance = Entity::instanceFromType($entity->type); + + if ($instance instanceof Page) { + $data['text'] = $data['description']; + unset($data['description']); + } + + $instance->forceFill($data); + $hydrated[] = $instance; + } + + if ($this->loadTags) { + $this->loadTagsIntoModels($hydrated); + } + + if ($this->loadParents) { + $this->loadParentsIntoModels($hydrated); + } + + return $hydrated; + } + + /** + * @param Entity[] $entities + */ + protected function loadTagsIntoModels(array $entities): void + { + $idsByType = []; + $entityMap = []; + foreach ($entities as $entity) { + if (!isset($idsByType[$entity->type])) { + $idsByType[$entity->type] = []; + } + $idsByType[$entity->type][] = $entity->id; + $entityMap[$entity->type . ':' . $entity->id] = $entity; + } + + $query = Tag::query(); + foreach ($idsByType as $type => $ids) { + $query->orWhere(function ($query) use ($type, $ids) { + $query->where('entity_type', '=', $type) + ->whereIn('entity_id', $ids); + }); + } + + $tags = empty($idsByType) ? [] : $query->get()->all(); + $tagMap = []; + foreach ($tags as $tag) { + $key = $tag->entity_type . ':' . $tag->entity_id; + if (!isset($tagMap[$key])) { + $tagMap[$key] = []; + } + $tagMap[$key][] = $tag; + } + + foreach ($entityMap as $key => $entity) { + $entityTags = new Collection($tagMap[$key] ?? []); + $entity->setRelation('tags', $entityTags); + } + } + + /** + * @param Entity[] $entities + */ + protected function loadParentsIntoModels(array $entities): void + { + $parentsByType = ['book' => [], 'chapter' => []]; + + foreach ($entities as $entity) { + if ($entity->getAttribute('book_id') !== null) { + $parentsByType['book'][] = $entity->getAttribute('book_id'); + } + if ($entity->getAttribute('chapter_id') !== null) { + $parentsByType['chapter'][] = $entity->getAttribute('chapter_id'); + } + } + + // TODO - Inject in? + $queries = app()->make(EntityQueries::class); + + $parentQuery = $queries->visibleForList(); + $filtered = count($parentsByType['book']) > 0 || count($parentsByType['chapter']) > 0; + $parentQuery = $parentQuery->where(function ($query) use ($parentsByType) { + foreach ($parentsByType as $type => $ids) { + if (count($ids) > 0) { + $query = $query->orWhere(function ($query) use ($type, $ids) { + $query->where('type', '=', $type) + ->whereIn('id', $ids); + }); + } + } + }); + + $parents = $filtered ? (new EntityHydrator($parentQuery->get()->all()))->hydrate() : []; + $parentMap = []; + foreach ($parents as $parent) { + $parentMap[$parent->type . ':' . $parent->id] = $parent; + } + + foreach ($entities as $entity) { + if ($entity instanceof Page || $entity instanceof Chapter) { + $key = 'book:' . $entity->getAttribute('book_id'); + $entity->setRelation('book', $parentMap[$key] ?? null); + } + if ($entity instanceof Page) { + $key = 'chapter:' . $entity->getAttribute('chapter_id'); + $entity->setRelation('chapter', $parentMap[$key] ?? null); + } + } + } +} diff --git a/app/Entities/Tools/MixedEntityListLoader.php b/app/Entities/Tools/MixedEntityListLoader.php index 0a0f224d8..9987cc061 100644 --- a/app/Entities/Tools/MixedEntityListLoader.php +++ b/app/Entities/Tools/MixedEntityListLoader.php @@ -54,7 +54,7 @@ class MixedEntityListLoader $modelMap = []; foreach ($idsByType as $type => $ids) { - $base = $withContents ? $this->queries->visibleForContent($type) : $this->queries->visibleForList($type); + $base = $withContents ? $this->queries->visibleForContentForType($type) : $this->queries->visibleForListForType($type); $models = $base->whereIn('id', $ids) ->with($eagerLoadParents ? $this->getRelationsToEagerLoad($type) : []) ->get(); diff --git a/app/Search/SearchRunner.php b/app/Search/SearchRunner.php index a1ffeee50..72ebe96bb 100644 --- a/app/Search/SearchRunner.php +++ b/app/Search/SearchRunner.php @@ -4,16 +4,16 @@ namespace BookStack\Search; use BookStack\Entities\EntityProvider; use BookStack\Entities\Models\Entity; -use BookStack\Entities\Models\Page; use BookStack\Entities\Queries\EntityQueries; +use BookStack\Entities\Tools\EntityHydrator; use BookStack\Permissions\PermissionApplicator; use BookStack\Search\Options\TagSearchOption; use BookStack\Users\Models\User; use Illuminate\Database\Connection; use Illuminate\Database\Eloquent\Builder as EloquentBuilder; use Illuminate\Database\Eloquent\Collection as EloquentCollection; -use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Query\Builder; +use Illuminate\Database\Query\JoinClause; use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; @@ -22,7 +22,7 @@ use WeakMap; class SearchRunner { /** - * Retain a cache of score adjusted terms for specific search options. + * Retain a cache of score-adjusted terms for specific search options. */ protected WeakMap $termAdjustmentCache; @@ -38,6 +38,7 @@ class SearchRunner * Search all entities in the system. * The provided count is for each entity to search, * Total returned could be larger and not guaranteed. + * // TODO - Update this comment * * @return array{total: int, count: int, has_more: bool, results: Collection} */ @@ -53,26 +54,12 @@ class SearchRunner $entityTypesToSearch = explode('|', $filterMap['type']); } - $results = collect(); - $total = 0; - $hasMore = false; + $searchQuery = $this->buildQuery($searchOpts, $entityTypesToSearch); + $total = $searchQuery->count(); + $results = $this->getPageOfDataFromQuery($searchQuery, $page, $count); - foreach ($entityTypesToSearch as $entityType) { - if (!in_array($entityType, $entityTypes)) { - continue; - } - - $searchQuery = $this->buildQuery($searchOpts, $entityType); - $entityTotal = $searchQuery->count(); - $searchResults = $this->getPageOfDataFromQuery($searchQuery, $entityType, $page, $count); - - if ($entityTotal > ($page * $count)) { - $hasMore = true; - } - - $total += $entityTotal; - $results = $results->merge($searchResults); - } + // TODO - Pagination? + $hasMore = ($total > ($page * $count)); return [ 'total' => $total, @@ -119,46 +106,41 @@ class SearchRunner /** * Get a page of result data from the given query based on the provided page parameters. */ - protected function getPageOfDataFromQuery(EloquentBuilder $query, string $entityType, int $page = 1, int $count = 20): EloquentCollection + protected function getPageOfDataFromQuery(EloquentBuilder $query, int $page, int $count): Collection { - $relations = ['tags']; - - if ($entityType === 'page' || $entityType === 'chapter') { - $relations['book'] = function (BelongsTo $query) { - $query->scopes('visible'); - }; - } - - if ($entityType === 'page') { - $relations['chapter'] = function (BelongsTo $query) { - $query->scopes('visible'); - }; - } - - return $query->clone() - ->with(array_filter($relations)) + $entities = $query->clone() +// ->with(array_filter($relations)) ->skip(($page - 1) * $count) ->take($count) ->get(); + + $hydrated = (new EntityHydrator($entities->all(), true, true))->hydrate(); + + // TODO - Load in books for pages/chapters efficiently (scoped to visible) + // TODO - Load in chapters for pages efficiently (scoped to visible) + // TODO - Load in tags efficiently + + return collect($hydrated); } /** * Create a search query for an entity. + * @param string[] $entityTypes */ - protected function buildQuery(SearchOptions $searchOpts, string $entityType): EloquentBuilder + protected function buildQuery(SearchOptions $searchOpts, array $entityTypes): EloquentBuilder { - $entityModelInstance = $this->entityProvider->get($entityType); - $entityQuery = $this->entityQueries->visibleForList($entityType); + $entityQuery = $this->entityQueries->visibleForList() + ->whereIn('type', $entityTypes); // Handle normal search terms - $this->applyTermSearch($entityQuery, $searchOpts, $entityType); + $this->applyTermSearch($entityQuery, $searchOpts, $entityTypes); // Handle exact term matching foreach ($searchOpts->exacts->all() as $exact) { - $filter = function (EloquentBuilder $query) use ($exact, $entityModelInstance) { + $filter = function (EloquentBuilder $query) use ($exact) { $inputTerm = str_replace('\\', '\\\\', $exact->value); $query->where('name', 'like', '%' . $inputTerm . '%') - ->orWhere($entityModelInstance->textField, 'like', '%' . $inputTerm . '%'); + ->orWhere('description', 'like', '%' . $inputTerm . '%'); }; $exact->negated ? $entityQuery->whereNot($filter) : $entityQuery->where($filter); @@ -173,7 +155,7 @@ class SearchRunner foreach ($searchOpts->filters->all() as $filterOption) { $functionName = Str::camel('filter_' . $filterOption->getKey()); if (method_exists($this, $functionName)) { - $this->$functionName($entityQuery, $entityModelInstance, $filterOption->value, $filterOption->negated); + $this->$functionName($entityQuery, $filterOption->value, $filterOption->negated); } } @@ -183,7 +165,7 @@ class SearchRunner /** * For the given search query, apply the queries for handling the regular search terms. */ - protected function applyTermSearch(EloquentBuilder $entityQuery, SearchOptions $options, string $entityType): void + protected function applyTermSearch(EloquentBuilder $entityQuery, SearchOptions $options, array $entityTypes): void { $terms = $options->searches->toValueArray(); if (count($terms) === 0) { @@ -200,8 +182,6 @@ class SearchRunner ]); $subQuery->addBinding($scoreSelect['bindings'], 'select'); - - $subQuery->where('entity_type', '=', $entityType); $subQuery->where(function (Builder $query) use ($terms) { foreach ($terms as $inputTerm) { $escapedTerm = str_replace('\\', '\\\\', $inputTerm); @@ -210,7 +190,10 @@ class SearchRunner }); $subQuery->groupBy('entity_type', 'entity_id'); - $entityQuery->joinSub($subQuery, 's', 'id', '=', 'entity_id'); + $entityQuery->joinSub($subQuery, 's', function (JoinClause $join) { + $join->on('s.entity_id', '=', 'entities.id') + ->on('s.entity_type', '=', 'entities.type'); + }); $entityQuery->addSelect('s.score'); $entityQuery->orderBy('score', 'desc'); } @@ -350,31 +333,31 @@ class SearchRunner /** * Custom entity search filters. */ - protected function filterUpdatedAfter(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterUpdatedAfter(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'updated_at', '>=', $date); } - protected function filterUpdatedBefore(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterUpdatedBefore(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'updated_at', '<', $date); } - protected function filterCreatedAfter(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterCreatedAfter(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'created_at', '>=', $date); } - protected function filterCreatedBefore(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterCreatedBefore(EloquentBuilder $query, string $input, bool $negated) { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'created_at', '<', $date); } - protected function filterCreatedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterCreatedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -383,7 +366,7 @@ class SearchRunner } } - protected function filterUpdatedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterUpdatedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -392,7 +375,7 @@ class SearchRunner } } - protected function filterOwnedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterOwnedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -401,27 +384,27 @@ class SearchRunner } } - protected function filterInName(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInName(EloquentBuilder $query, string $input, bool $negated) { $this->applyNegatableWhere($query, $negated, 'name', 'like', '%' . $input . '%'); } - protected function filterInTitle(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInTitle(EloquentBuilder $query, string $input, bool $negated) { - $this->filterInName($query, $model, $input, $negated); + $this->filterInName($query, $input, $negated); } - protected function filterInBody(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInBody(EloquentBuilder $query, string $input, bool $negated) { - $this->applyNegatableWhere($query, $negated, $model->textField, 'like', '%' . $input . '%'); + $this->applyNegatableWhere($query, $negated, 'description', 'like', '%' . $input . '%'); } - protected function filterIsRestricted(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterIsRestricted(EloquentBuilder $query, string $input, bool $negated) { $negated ? $query->whereDoesntHave('permissions') : $query->whereHas('permissions'); } - protected function filterViewedByMe(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterViewedByMe(EloquentBuilder $query, string $input, bool $negated) { $filter = function ($query) { $query->where('user_id', '=', user()->id); @@ -430,7 +413,7 @@ class SearchRunner $negated ? $query->whereDoesntHave('views', $filter) : $query->whereHas('views', $filter); } - protected function filterNotViewedByMe(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterNotViewedByMe(EloquentBuilder $query, string $input, bool $negated) { $filter = function ($query) { $query->where('user_id', '=', user()->id); @@ -439,31 +422,30 @@ class SearchRunner $negated ? $query->whereHas('views', $filter) : $query->whereDoesntHave('views', $filter); } - protected function filterIsTemplate(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterIsTemplate(EloquentBuilder $query, string $input, bool $negated) { - if ($model instanceof Page) { - $this->applyNegatableWhere($query, $negated, 'template', '=', true); - } + $this->applyNegatableWhere($query, $negated, 'template', '=', true); } - protected function filterSortBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterSortBy(EloquentBuilder $query, string $input, bool $negated) { $functionName = Str::camel('sort_by_' . $input); if (method_exists($this, $functionName)) { - $this->$functionName($query, $model, $negated); + $this->$functionName($query, $negated); } } /** * Sorting filter options. */ - protected function sortByLastCommented(EloquentBuilder $query, Entity $model, bool $negated) + protected function sortByLastCommented(EloquentBuilder $query, bool $negated) { $commentsTable = DB::getTablePrefix() . 'comments'; - $morphClass = str_replace('\\', '\\\\', $model->getMorphClass()); - $commentQuery = DB::raw('(SELECT c1.commentable_id, c1.commentable_type, c1.created_at as last_commented FROM ' . $commentsTable . ' c1 LEFT JOIN ' . $commentsTable . ' c2 ON (c1.commentable_id = c2.commentable_id AND c1.commentable_type = c2.commentable_type AND c1.created_at < c2.created_at) WHERE c1.commentable_type = \'' . $morphClass . '\' AND c2.created_at IS NULL) as comments'); + $commentQuery = DB::raw('(SELECT c1.commentable_id, c1.commentable_type, c1.created_at as last_commented FROM ' . $commentsTable . ' c1 LEFT JOIN ' . $commentsTable . ' c2 ON (c1.commentable_id = c2.commentable_id AND c1.commentable_type = c2.commentable_type AND c1.created_at < c2.created_at) WHERE c2.created_at IS NULL) as comments'); - $query->join($commentQuery, $model->getTable() . '.id', '=', DB::raw('comments.commentable_id')) - ->orderBy('last_commented', $negated ? 'asc' : 'desc'); + $query->join($commentQuery, function (JoinClause $join) { + $join->on('entities.id', '=', 'comments.commentable_id') + ->on('entities.type', '=', 'comments.commentable_type'); + })->orderBy('last_commented', $negated ? 'asc' : 'desc'); } }