From 0b265733142a21de91ed90d0e197c48b0561bb1e Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Mon, 27 Oct 2025 17:23:15 +0000 Subject: [PATCH 1/4] Search: Started work to make search result size consistent --- app/Entities/Models/Entity.php | 13 ++ app/Entities/Models/EntityTable.php | 37 +++++ app/Entities/Queries/EntityQueries.php | 26 +++- app/Entities/Tools/EntityHydrator.php | 151 +++++++++++++++++++ app/Entities/Tools/MixedEntityListLoader.php | 2 +- app/Search/SearchRunner.php | 134 +++++++--------- 6 files changed, 284 insertions(+), 79 deletions(-) create mode 100644 app/Entities/Models/EntityTable.php create mode 100644 app/Entities/Tools/EntityHydrator.php diff --git a/app/Entities/Models/Entity.php b/app/Entities/Models/Entity.php index 77393cbbc..b47a029ec 100644 --- a/app/Entities/Models/Entity.php +++ b/app/Entities/Models/Entity.php @@ -471,4 +471,17 @@ abstract class Entity extends Model implements return $contentFields; } + + /** + * Create a new instance for the given entity type. + */ + public static function instanceFromType(string $type): self + { + return match ($type) { + 'page' => new Page(), + 'chapter' => new Chapter(), + 'book' => new Book(), + 'bookshelf' => new Bookshelf(), + }; + } } diff --git a/app/Entities/Models/EntityTable.php b/app/Entities/Models/EntityTable.php new file mode 100644 index 000000000..50112a8e9 --- /dev/null +++ b/app/Entities/Models/EntityTable.php @@ -0,0 +1,37 @@ +make(PermissionApplicator::class)->restrictEntityQuery($query); + } + + /** + * Get the entity jointPermissions this is connected to. + */ + public function jointPermissions(): HasMany + { + return $this->hasMany(JointPermission::class, 'entity_id')->whereColumn('entity_type', '=', 'entities.type'); + } +} diff --git a/app/Entities/Queries/EntityQueries.php b/app/Entities/Queries/EntityQueries.php index a7a037916..c27cc61cc 100644 --- a/app/Entities/Queries/EntityQueries.php +++ b/app/Entities/Queries/EntityQueries.php @@ -3,7 +3,10 @@ namespace BookStack\Entities\Queries; use BookStack\Entities\Models\Entity; +use BookStack\Entities\Models\EntityTable; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Query\JoinClause; +use Illuminate\Support\Facades\DB; use InvalidArgumentException; class EntityQueries @@ -32,12 +35,31 @@ class EntityQueries return $queries->findVisibleById($entityId); } + /** + * Start a query across all entity types. + * Combines the description/text fields into a single 'description' field. + * @return Builder + */ + public function visibleForList(): Builder + { + $rawDescriptionField = DB::raw('COALESCE(description, text) as description'); + return EntityTable::query()->scopes('visible') + ->select(['id', 'type', 'name', 'slug', 'book_id', 'chapter_id', 'created_at', 'updated_at', 'draft', $rawDescriptionField]) + ->leftJoin('entity_container_data', function (JoinClause $join) { + $join->on('entity_container_data.entity_id', '=', 'entities.id') + ->on('entity_container_data.entity_type', '=', 'entities.type'); + })->leftJoin('entity_page_data', function (JoinClause $join) { + $join->on('entity_page_data.page_id', '=', 'entities.id') + ->where('entities.type', '=', 'page'); + }); + } + /** * Start a query of visible entities of the given type, * suitable for listing display. * @return Builder */ - public function visibleForList(string $entityType): Builder + public function visibleForListForType(string $entityType): Builder { $queries = $this->getQueriesForType($entityType); return $queries->visibleForList(); @@ -48,7 +70,7 @@ class EntityQueries * suitable for using the contents of the items. * @return Builder */ - public function visibleForContent(string $entityType): Builder + public function visibleForContentForType(string $entityType): Builder { $queries = $this->getQueriesForType($entityType); return $queries->visibleForContent(); diff --git a/app/Entities/Tools/EntityHydrator.php b/app/Entities/Tools/EntityHydrator.php new file mode 100644 index 000000000..fbfc8762e --- /dev/null +++ b/app/Entities/Tools/EntityHydrator.php @@ -0,0 +1,151 @@ +entities = $entities; + $this->loadTags = $loadTags; + $this->loadParents = $loadParents; + } + + /** + * Hydrate the entities of this hydrator to return a list of entities represented + * in their original intended models. + * @return Entity[] + */ + public function hydrate(): array + { + $hydrated = []; + + foreach ($this->entities as $entity) { + $data = $entity->toArray(); + $instance = Entity::instanceFromType($entity->type); + + if ($instance instanceof Page) { + $data['text'] = $data['description']; + unset($data['description']); + } + + $instance->forceFill($data); + $hydrated[] = $instance; + } + + if ($this->loadTags) { + $this->loadTagsIntoModels($hydrated); + } + + if ($this->loadParents) { + $this->loadParentsIntoModels($hydrated); + } + + return $hydrated; + } + + /** + * @param Entity[] $entities + */ + protected function loadTagsIntoModels(array $entities): void + { + $idsByType = []; + $entityMap = []; + foreach ($entities as $entity) { + if (!isset($idsByType[$entity->type])) { + $idsByType[$entity->type] = []; + } + $idsByType[$entity->type][] = $entity->id; + $entityMap[$entity->type . ':' . $entity->id] = $entity; + } + + $query = Tag::query(); + foreach ($idsByType as $type => $ids) { + $query->orWhere(function ($query) use ($type, $ids) { + $query->where('entity_type', '=', $type) + ->whereIn('entity_id', $ids); + }); + } + + $tags = empty($idsByType) ? [] : $query->get()->all(); + $tagMap = []; + foreach ($tags as $tag) { + $key = $tag->entity_type . ':' . $tag->entity_id; + if (!isset($tagMap[$key])) { + $tagMap[$key] = []; + } + $tagMap[$key][] = $tag; + } + + foreach ($entityMap as $key => $entity) { + $entityTags = new Collection($tagMap[$key] ?? []); + $entity->setRelation('tags', $entityTags); + } + } + + /** + * @param Entity[] $entities + */ + protected function loadParentsIntoModels(array $entities): void + { + $parentsByType = ['book' => [], 'chapter' => []]; + + foreach ($entities as $entity) { + if ($entity->getAttribute('book_id') !== null) { + $parentsByType['book'][] = $entity->getAttribute('book_id'); + } + if ($entity->getAttribute('chapter_id') !== null) { + $parentsByType['chapter'][] = $entity->getAttribute('chapter_id'); + } + } + + // TODO - Inject in? + $queries = app()->make(EntityQueries::class); + + $parentQuery = $queries->visibleForList(); + $filtered = count($parentsByType['book']) > 0 || count($parentsByType['chapter']) > 0; + $parentQuery = $parentQuery->where(function ($query) use ($parentsByType) { + foreach ($parentsByType as $type => $ids) { + if (count($ids) > 0) { + $query = $query->orWhere(function ($query) use ($type, $ids) { + $query->where('type', '=', $type) + ->whereIn('id', $ids); + }); + } + } + }); + + $parents = $filtered ? (new EntityHydrator($parentQuery->get()->all()))->hydrate() : []; + $parentMap = []; + foreach ($parents as $parent) { + $parentMap[$parent->type . ':' . $parent->id] = $parent; + } + + foreach ($entities as $entity) { + if ($entity instanceof Page || $entity instanceof Chapter) { + $key = 'book:' . $entity->getAttribute('book_id'); + $entity->setRelation('book', $parentMap[$key] ?? null); + } + if ($entity instanceof Page) { + $key = 'chapter:' . $entity->getAttribute('chapter_id'); + $entity->setRelation('chapter', $parentMap[$key] ?? null); + } + } + } +} diff --git a/app/Entities/Tools/MixedEntityListLoader.php b/app/Entities/Tools/MixedEntityListLoader.php index 0a0f224d8..9987cc061 100644 --- a/app/Entities/Tools/MixedEntityListLoader.php +++ b/app/Entities/Tools/MixedEntityListLoader.php @@ -54,7 +54,7 @@ class MixedEntityListLoader $modelMap = []; foreach ($idsByType as $type => $ids) { - $base = $withContents ? $this->queries->visibleForContent($type) : $this->queries->visibleForList($type); + $base = $withContents ? $this->queries->visibleForContentForType($type) : $this->queries->visibleForListForType($type); $models = $base->whereIn('id', $ids) ->with($eagerLoadParents ? $this->getRelationsToEagerLoad($type) : []) ->get(); diff --git a/app/Search/SearchRunner.php b/app/Search/SearchRunner.php index a1ffeee50..72ebe96bb 100644 --- a/app/Search/SearchRunner.php +++ b/app/Search/SearchRunner.php @@ -4,16 +4,16 @@ namespace BookStack\Search; use BookStack\Entities\EntityProvider; use BookStack\Entities\Models\Entity; -use BookStack\Entities\Models\Page; use BookStack\Entities\Queries\EntityQueries; +use BookStack\Entities\Tools\EntityHydrator; use BookStack\Permissions\PermissionApplicator; use BookStack\Search\Options\TagSearchOption; use BookStack\Users\Models\User; use Illuminate\Database\Connection; use Illuminate\Database\Eloquent\Builder as EloquentBuilder; use Illuminate\Database\Eloquent\Collection as EloquentCollection; -use Illuminate\Database\Eloquent\Relations\BelongsTo; use Illuminate\Database\Query\Builder; +use Illuminate\Database\Query\JoinClause; use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Str; @@ -22,7 +22,7 @@ use WeakMap; class SearchRunner { /** - * Retain a cache of score adjusted terms for specific search options. + * Retain a cache of score-adjusted terms for specific search options. */ protected WeakMap $termAdjustmentCache; @@ -38,6 +38,7 @@ class SearchRunner * Search all entities in the system. * The provided count is for each entity to search, * Total returned could be larger and not guaranteed. + * // TODO - Update this comment * * @return array{total: int, count: int, has_more: bool, results: Collection} */ @@ -53,26 +54,12 @@ class SearchRunner $entityTypesToSearch = explode('|', $filterMap['type']); } - $results = collect(); - $total = 0; - $hasMore = false; + $searchQuery = $this->buildQuery($searchOpts, $entityTypesToSearch); + $total = $searchQuery->count(); + $results = $this->getPageOfDataFromQuery($searchQuery, $page, $count); - foreach ($entityTypesToSearch as $entityType) { - if (!in_array($entityType, $entityTypes)) { - continue; - } - - $searchQuery = $this->buildQuery($searchOpts, $entityType); - $entityTotal = $searchQuery->count(); - $searchResults = $this->getPageOfDataFromQuery($searchQuery, $entityType, $page, $count); - - if ($entityTotal > ($page * $count)) { - $hasMore = true; - } - - $total += $entityTotal; - $results = $results->merge($searchResults); - } + // TODO - Pagination? + $hasMore = ($total > ($page * $count)); return [ 'total' => $total, @@ -119,46 +106,41 @@ class SearchRunner /** * Get a page of result data from the given query based on the provided page parameters. */ - protected function getPageOfDataFromQuery(EloquentBuilder $query, string $entityType, int $page = 1, int $count = 20): EloquentCollection + protected function getPageOfDataFromQuery(EloquentBuilder $query, int $page, int $count): Collection { - $relations = ['tags']; - - if ($entityType === 'page' || $entityType === 'chapter') { - $relations['book'] = function (BelongsTo $query) { - $query->scopes('visible'); - }; - } - - if ($entityType === 'page') { - $relations['chapter'] = function (BelongsTo $query) { - $query->scopes('visible'); - }; - } - - return $query->clone() - ->with(array_filter($relations)) + $entities = $query->clone() +// ->with(array_filter($relations)) ->skip(($page - 1) * $count) ->take($count) ->get(); + + $hydrated = (new EntityHydrator($entities->all(), true, true))->hydrate(); + + // TODO - Load in books for pages/chapters efficiently (scoped to visible) + // TODO - Load in chapters for pages efficiently (scoped to visible) + // TODO - Load in tags efficiently + + return collect($hydrated); } /** * Create a search query for an entity. + * @param string[] $entityTypes */ - protected function buildQuery(SearchOptions $searchOpts, string $entityType): EloquentBuilder + protected function buildQuery(SearchOptions $searchOpts, array $entityTypes): EloquentBuilder { - $entityModelInstance = $this->entityProvider->get($entityType); - $entityQuery = $this->entityQueries->visibleForList($entityType); + $entityQuery = $this->entityQueries->visibleForList() + ->whereIn('type', $entityTypes); // Handle normal search terms - $this->applyTermSearch($entityQuery, $searchOpts, $entityType); + $this->applyTermSearch($entityQuery, $searchOpts, $entityTypes); // Handle exact term matching foreach ($searchOpts->exacts->all() as $exact) { - $filter = function (EloquentBuilder $query) use ($exact, $entityModelInstance) { + $filter = function (EloquentBuilder $query) use ($exact) { $inputTerm = str_replace('\\', '\\\\', $exact->value); $query->where('name', 'like', '%' . $inputTerm . '%') - ->orWhere($entityModelInstance->textField, 'like', '%' . $inputTerm . '%'); + ->orWhere('description', 'like', '%' . $inputTerm . '%'); }; $exact->negated ? $entityQuery->whereNot($filter) : $entityQuery->where($filter); @@ -173,7 +155,7 @@ class SearchRunner foreach ($searchOpts->filters->all() as $filterOption) { $functionName = Str::camel('filter_' . $filterOption->getKey()); if (method_exists($this, $functionName)) { - $this->$functionName($entityQuery, $entityModelInstance, $filterOption->value, $filterOption->negated); + $this->$functionName($entityQuery, $filterOption->value, $filterOption->negated); } } @@ -183,7 +165,7 @@ class SearchRunner /** * For the given search query, apply the queries for handling the regular search terms. */ - protected function applyTermSearch(EloquentBuilder $entityQuery, SearchOptions $options, string $entityType): void + protected function applyTermSearch(EloquentBuilder $entityQuery, SearchOptions $options, array $entityTypes): void { $terms = $options->searches->toValueArray(); if (count($terms) === 0) { @@ -200,8 +182,6 @@ class SearchRunner ]); $subQuery->addBinding($scoreSelect['bindings'], 'select'); - - $subQuery->where('entity_type', '=', $entityType); $subQuery->where(function (Builder $query) use ($terms) { foreach ($terms as $inputTerm) { $escapedTerm = str_replace('\\', '\\\\', $inputTerm); @@ -210,7 +190,10 @@ class SearchRunner }); $subQuery->groupBy('entity_type', 'entity_id'); - $entityQuery->joinSub($subQuery, 's', 'id', '=', 'entity_id'); + $entityQuery->joinSub($subQuery, 's', function (JoinClause $join) { + $join->on('s.entity_id', '=', 'entities.id') + ->on('s.entity_type', '=', 'entities.type'); + }); $entityQuery->addSelect('s.score'); $entityQuery->orderBy('score', 'desc'); } @@ -350,31 +333,31 @@ class SearchRunner /** * Custom entity search filters. */ - protected function filterUpdatedAfter(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterUpdatedAfter(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'updated_at', '>=', $date); } - protected function filterUpdatedBefore(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterUpdatedBefore(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'updated_at', '<', $date); } - protected function filterCreatedAfter(EloquentBuilder $query, Entity $model, string $input, bool $negated): void + protected function filterCreatedAfter(EloquentBuilder $query, string $input, bool $negated): void { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'created_at', '>=', $date); } - protected function filterCreatedBefore(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterCreatedBefore(EloquentBuilder $query, string $input, bool $negated) { $date = date_create($input); $this->applyNegatableWhere($query, $negated, 'created_at', '<', $date); } - protected function filterCreatedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterCreatedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -383,7 +366,7 @@ class SearchRunner } } - protected function filterUpdatedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterUpdatedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -392,7 +375,7 @@ class SearchRunner } } - protected function filterOwnedBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterOwnedBy(EloquentBuilder $query, string $input, bool $negated) { $userSlug = $input === 'me' ? user()->slug : trim($input); $user = User::query()->where('slug', '=', $userSlug)->first(['id']); @@ -401,27 +384,27 @@ class SearchRunner } } - protected function filterInName(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInName(EloquentBuilder $query, string $input, bool $negated) { $this->applyNegatableWhere($query, $negated, 'name', 'like', '%' . $input . '%'); } - protected function filterInTitle(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInTitle(EloquentBuilder $query, string $input, bool $negated) { - $this->filterInName($query, $model, $input, $negated); + $this->filterInName($query, $input, $negated); } - protected function filterInBody(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterInBody(EloquentBuilder $query, string $input, bool $negated) { - $this->applyNegatableWhere($query, $negated, $model->textField, 'like', '%' . $input . '%'); + $this->applyNegatableWhere($query, $negated, 'description', 'like', '%' . $input . '%'); } - protected function filterIsRestricted(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterIsRestricted(EloquentBuilder $query, string $input, bool $negated) { $negated ? $query->whereDoesntHave('permissions') : $query->whereHas('permissions'); } - protected function filterViewedByMe(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterViewedByMe(EloquentBuilder $query, string $input, bool $negated) { $filter = function ($query) { $query->where('user_id', '=', user()->id); @@ -430,7 +413,7 @@ class SearchRunner $negated ? $query->whereDoesntHave('views', $filter) : $query->whereHas('views', $filter); } - protected function filterNotViewedByMe(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterNotViewedByMe(EloquentBuilder $query, string $input, bool $negated) { $filter = function ($query) { $query->where('user_id', '=', user()->id); @@ -439,31 +422,30 @@ class SearchRunner $negated ? $query->whereHas('views', $filter) : $query->whereDoesntHave('views', $filter); } - protected function filterIsTemplate(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterIsTemplate(EloquentBuilder $query, string $input, bool $negated) { - if ($model instanceof Page) { - $this->applyNegatableWhere($query, $negated, 'template', '=', true); - } + $this->applyNegatableWhere($query, $negated, 'template', '=', true); } - protected function filterSortBy(EloquentBuilder $query, Entity $model, string $input, bool $negated) + protected function filterSortBy(EloquentBuilder $query, string $input, bool $negated) { $functionName = Str::camel('sort_by_' . $input); if (method_exists($this, $functionName)) { - $this->$functionName($query, $model, $negated); + $this->$functionName($query, $negated); } } /** * Sorting filter options. */ - protected function sortByLastCommented(EloquentBuilder $query, Entity $model, bool $negated) + protected function sortByLastCommented(EloquentBuilder $query, bool $negated) { $commentsTable = DB::getTablePrefix() . 'comments'; - $morphClass = str_replace('\\', '\\\\', $model->getMorphClass()); - $commentQuery = DB::raw('(SELECT c1.commentable_id, c1.commentable_type, c1.created_at as last_commented FROM ' . $commentsTable . ' c1 LEFT JOIN ' . $commentsTable . ' c2 ON (c1.commentable_id = c2.commentable_id AND c1.commentable_type = c2.commentable_type AND c1.created_at < c2.created_at) WHERE c1.commentable_type = \'' . $morphClass . '\' AND c2.created_at IS NULL) as comments'); + $commentQuery = DB::raw('(SELECT c1.commentable_id, c1.commentable_type, c1.created_at as last_commented FROM ' . $commentsTable . ' c1 LEFT JOIN ' . $commentsTable . ' c2 ON (c1.commentable_id = c2.commentable_id AND c1.commentable_type = c2.commentable_type AND c1.created_at < c2.created_at) WHERE c2.created_at IS NULL) as comments'); - $query->join($commentQuery, $model->getTable() . '.id', '=', DB::raw('comments.commentable_id')) - ->orderBy('last_commented', $negated ? 'asc' : 'desc'); + $query->join($commentQuery, function (JoinClause $join) { + $join->on('entities.id', '=', 'comments.commentable_id') + ->on('entities.type', '=', 'comments.commentable_type'); + })->orderBy('last_commented', $negated ? 'asc' : 'desc'); } } From f0303de2e5aa5a2e7b2bc805653c5cf3039ea290 Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Mon, 27 Oct 2025 18:02:54 +0000 Subject: [PATCH 2/4] Search: Improved result hydration performance --- app/Entities/Tools/EntityHydrator.php | 15 ++++++++------- app/Search/SearchRunner.php | 5 ----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/app/Entities/Tools/EntityHydrator.php b/app/Entities/Tools/EntityHydrator.php index fbfc8762e..cb6267423 100644 --- a/app/Entities/Tools/EntityHydrator.php +++ b/app/Entities/Tools/EntityHydrator.php @@ -37,7 +37,7 @@ class EntityHydrator $hydrated = []; foreach ($this->entities as $entity) { - $data = $entity->toArray(); + $data = $entity->getRawOriginal(); $instance = Entity::instanceFromType($entity->type); if ($instance instanceof Page) { @@ -45,7 +45,7 @@ class EntityHydrator unset($data['description']); } - $instance->forceFill($data); + $instance = $instance->setRawAttributes($data, true); $hydrated[] = $instance; } @@ -131,7 +131,8 @@ class EntityHydrator } }); - $parents = $filtered ? (new EntityHydrator($parentQuery->get()->all()))->hydrate() : []; + $parentModels = $filtered ? $parentQuery->get()->all() : []; + $parents = (new EntityHydrator($parentModels))->hydrate(); $parentMap = []; foreach ($parents as $parent) { $parentMap[$parent->type . ':' . $parent->id] = $parent; @@ -139,12 +140,12 @@ class EntityHydrator foreach ($entities as $entity) { if ($entity instanceof Page || $entity instanceof Chapter) { - $key = 'book:' . $entity->getAttribute('book_id'); - $entity->setRelation('book', $parentMap[$key] ?? null); + $key = 'book:' . $entity->getRawAttribute('book_id'); + $entity->setAttribute('book', $parentMap[$key] ?? null); } if ($entity instanceof Page) { - $key = 'chapter:' . $entity->getAttribute('chapter_id'); - $entity->setRelation('chapter', $parentMap[$key] ?? null); + $key = 'chapter:' . $entity->getRawAttribute('chapter_id'); + $entity->setAttribute('chapter', $parentMap[$key] ?? null); } } } diff --git a/app/Search/SearchRunner.php b/app/Search/SearchRunner.php index 72ebe96bb..4b19aceac 100644 --- a/app/Search/SearchRunner.php +++ b/app/Search/SearchRunner.php @@ -109,17 +109,12 @@ class SearchRunner protected function getPageOfDataFromQuery(EloquentBuilder $query, int $page, int $count): Collection { $entities = $query->clone() -// ->with(array_filter($relations)) ->skip(($page - 1) * $count) ->take($count) ->get(); $hydrated = (new EntityHydrator($entities->all(), true, true))->hydrate(); - // TODO - Load in books for pages/chapters efficiently (scoped to visible) - // TODO - Load in chapters for pages efficiently (scoped to visible) - // TODO - Load in tags efficiently - return collect($hydrated); } From 3fd25bd03e15a965213d61b5887b15ba3d172eef Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Tue, 28 Oct 2025 20:37:41 +0000 Subject: [PATCH 3/4] Search: Added pagination, updated other search uses Also updated hydrator to be created via injection. --- app/Entities/Tools/EntityHydrator.php | 32 +++++++++------------------ app/Search/SearchApiController.php | 9 ++++---- app/Search/SearchController.php | 14 +++++++----- app/Search/SearchRunner.php | 29 ++++++------------------ resources/views/search/all.blade.php | 6 +---- 5 files changed, 31 insertions(+), 59 deletions(-) diff --git a/app/Entities/Tools/EntityHydrator.php b/app/Entities/Tools/EntityHydrator.php index cb6267423..7cabc7ecb 100644 --- a/app/Entities/Tools/EntityHydrator.php +++ b/app/Entities/Tools/EntityHydrator.php @@ -12,31 +12,22 @@ use Illuminate\Database\Eloquent\Collection; class EntityHydrator { - /** - * @var EntityTable[] $entities - */ - protected array $entities; - - protected bool $loadTags = false; - protected bool $loadParents = false; - - public function __construct(array $entities, bool $loadTags = false, bool $loadParents = false) - { - $this->entities = $entities; - $this->loadTags = $loadTags; - $this->loadParents = $loadParents; + public function __construct( + protected EntityQueries $entityQueries, + ) { } /** * Hydrate the entities of this hydrator to return a list of entities represented * in their original intended models. + * @param EntityTable[] $entities * @return Entity[] */ - public function hydrate(): array + public function hydrate(array $entities, bool $loadTags = false, bool $loadParents = false): array { $hydrated = []; - foreach ($this->entities as $entity) { + foreach ($entities as $entity) { $data = $entity->getRawOriginal(); $instance = Entity::instanceFromType($entity->type); @@ -49,11 +40,11 @@ class EntityHydrator $hydrated[] = $instance; } - if ($this->loadTags) { + if ($loadTags) { $this->loadTagsIntoModels($hydrated); } - if ($this->loadParents) { + if ($loadParents) { $this->loadParentsIntoModels($hydrated); } @@ -115,10 +106,7 @@ class EntityHydrator } } - // TODO - Inject in? - $queries = app()->make(EntityQueries::class); - - $parentQuery = $queries->visibleForList(); + $parentQuery = $this->entityQueries->visibleForList(); $filtered = count($parentsByType['book']) > 0 || count($parentsByType['chapter']) > 0; $parentQuery = $parentQuery->where(function ($query) use ($parentsByType) { foreach ($parentsByType as $type => $ids) { @@ -132,7 +120,7 @@ class EntityHydrator }); $parentModels = $filtered ? $parentQuery->get()->all() : []; - $parents = (new EntityHydrator($parentModels))->hydrate(); + $parents = $this->hydrate($parentModels); $parentMap = []; foreach ($parents as $parent) { $parentMap[$parent->type . ':' . $parent->id] = $parent; diff --git a/app/Search/SearchApiController.php b/app/Search/SearchApiController.php index cd4a14a39..5de7a5110 100644 --- a/app/Search/SearchApiController.php +++ b/app/Search/SearchApiController.php @@ -1,10 +1,13 @@ validate($request, $this->rules['all']); diff --git a/app/Search/SearchController.php b/app/Search/SearchController.php index 2fce6a3d5..9586beffb 100644 --- a/app/Search/SearchController.php +++ b/app/Search/SearchController.php @@ -7,6 +7,7 @@ use BookStack\Entities\Queries\QueryPopular; use BookStack\Entities\Tools\SiblingFetcher; use BookStack\Http\Controller; use Illuminate\Http\Request; +use Illuminate\Pagination\LengthAwarePaginator; class SearchController extends Controller { @@ -23,20 +24,21 @@ class SearchController extends Controller { $searchOpts = SearchOptions::fromRequest($request); $fullSearchString = $searchOpts->toString(); - $this->setPageTitle(trans('entities.search_for_term', ['term' => $fullSearchString])); - $page = intval($request->get('page', '0')) ?: 1; - $nextPageLink = url('/search?term=' . urlencode($fullSearchString) . '&page=' . ($page + 1)); $results = $this->searchRunner->searchEntities($searchOpts, 'all', $page, 20); $formatter->format($results['results']->all(), $searchOpts); + $paginator = new LengthAwarePaginator($results['results'], $results['total'], 20, $page); + $paginator->setPath('/search'); + $paginator->appends($request->except('page')); + + $this->setPageTitle(trans('entities.search_for_term', ['term' => $fullSearchString])); return view('search.all', [ 'entities' => $results['results'], 'totalResults' => $results['total'], + 'paginator' => $paginator, 'searchTerm' => $fullSearchString, - 'hasNextPage' => $results['has_more'], - 'nextPageLink' => $nextPageLink, 'options' => $searchOpts, ]); } @@ -128,7 +130,7 @@ class SearchController extends Controller } /** - * Search siblings items in the system. + * Search sibling items in the system. */ public function searchSiblings(Request $request, SiblingFetcher $siblingFetcher) { diff --git a/app/Search/SearchRunner.php b/app/Search/SearchRunner.php index 4b19aceac..0128e3804 100644 --- a/app/Search/SearchRunner.php +++ b/app/Search/SearchRunner.php @@ -11,7 +11,6 @@ use BookStack\Search\Options\TagSearchOption; use BookStack\Users\Models\User; use Illuminate\Database\Connection; use Illuminate\Database\Eloquent\Builder as EloquentBuilder; -use Illuminate\Database\Eloquent\Collection as EloquentCollection; use Illuminate\Database\Query\Builder; use Illuminate\Database\Query\JoinClause; use Illuminate\Support\Collection; @@ -30,17 +29,15 @@ class SearchRunner protected EntityProvider $entityProvider, protected PermissionApplicator $permissions, protected EntityQueries $entityQueries, + protected EntityHydrator $entityHydrator, ) { $this->termAdjustmentCache = new WeakMap(); } /** * Search all entities in the system. - * The provided count is for each entity to search, - * Total returned could be larger and not guaranteed. - * // TODO - Update this comment * - * @return array{total: int, count: int, has_more: bool, results: Collection} + * @return array{total: int, results: Collection} */ public function searchEntities(SearchOptions $searchOpts, string $entityType = 'all', int $page = 1, int $count = 20): array { @@ -58,14 +55,9 @@ class SearchRunner $total = $searchQuery->count(); $results = $this->getPageOfDataFromQuery($searchQuery, $page, $count); - // TODO - Pagination? - $hasMore = ($total > ($page * $count)); - return [ 'total' => $total, - 'count' => count($results), - 'has_more' => $hasMore, - 'results' => $results->sortByDesc('score')->values(), + 'results' => $results->values(), ]; } @@ -79,15 +71,8 @@ class SearchRunner $filterMap = $opts->filters->toValueMap(); $entityTypesToSearch = isset($filterMap['type']) ? explode('|', $filterMap['type']) : $entityTypes; - $results = collect(); - foreach ($entityTypesToSearch as $entityType) { - if (!in_array($entityType, $entityTypes)) { - continue; - } - - $search = $this->buildQuery($opts, $entityType)->where('book_id', '=', $bookId)->take(20)->get(); - $results = $results->merge($search); - } + $filteredTypes = array_intersect($entityTypesToSearch, $entityTypes); + $results = $this->buildQuery($opts, $filteredTypes)->where('book_id', '=', $bookId)->take(20)->get(); return $results->sortByDesc('score')->take(20); } @@ -98,7 +83,7 @@ class SearchRunner public function searchChapter(int $chapterId, string $searchString): Collection { $opts = SearchOptions::fromString($searchString); - $pages = $this->buildQuery($opts, 'page')->where('chapter_id', '=', $chapterId)->take(20)->get(); + $pages = $this->buildQuery($opts, ['page'])->where('chapter_id', '=', $chapterId)->take(20)->get(); return $pages->sortByDesc('score'); } @@ -113,7 +98,7 @@ class SearchRunner ->take($count) ->get(); - $hydrated = (new EntityHydrator($entities->all(), true, true))->hydrate(); + $hydrated = $this->entityHydrator->hydrate($entities->all(), true, true); return collect($hydrated); } diff --git a/resources/views/search/all.blade.php b/resources/views/search/all.blade.php index ad437604b..48250816f 100644 --- a/resources/views/search/all.blade.php +++ b/resources/views/search/all.blade.php @@ -87,11 +87,7 @@ @include('entities.list', ['entities' => $entities, 'showPath' => true, 'showTags' => true]) - @if($hasNextPage) -
- {{ trans('entities.search_more') }} -
- @endif + {{ $paginator->render() }} From 751934c84a3e4c0482ca8086997c485ee6d6f077 Mon Sep 17 00:00:00 2001 From: Dan Brown Date: Wed, 29 Oct 2025 12:59:34 +0000 Subject: [PATCH 4/4] Search: Tested changes to single-table search Updated filters to use single table where needed. --- app/Entities/Models/EntityTable.php | 34 +++++++++++++++++++++++++- app/Entities/Queries/EntityQueries.php | 9 ++++++- app/Entities/Tools/EntityHydrator.php | 4 +-- app/Search/SearchRunner.php | 18 ++++++++------ tests/Api/SearchApiTest.php | 1 + tests/Search/EntitySearchTest.php | 6 +++++ 6 files changed, 61 insertions(+), 11 deletions(-) diff --git a/app/Entities/Models/EntityTable.php b/app/Entities/Models/EntityTable.php index 50112a8e9..5780162d1 100644 --- a/app/Entities/Models/EntityTable.php +++ b/app/Entities/Models/EntityTable.php @@ -2,11 +2,15 @@ namespace BookStack\Entities\Models; +use BookStack\Activity\Models\Tag; +use BookStack\Activity\Models\View; use BookStack\App\Model; +use BookStack\Permissions\Models\EntityPermission; use BookStack\Permissions\Models\JointPermission; use BookStack\Permissions\PermissionApplicator; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Relations\HasMany; +use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Eloquent\SoftDeletes; /** @@ -32,6 +36,34 @@ class EntityTable extends Model */ public function jointPermissions(): HasMany { - return $this->hasMany(JointPermission::class, 'entity_id')->whereColumn('entity_type', '=', 'entities.type'); + return $this->hasMany(JointPermission::class, 'entity_id') + ->whereColumn('entity_type', '=', 'entities.type'); + } + + /** + * Get the Tags that have been assigned to entities. + */ + public function tags(): HasMany + { + return $this->hasMany(Tag::class, 'entity_id') + ->whereColumn('entity_type', '=', 'entities.type'); + } + + /** + * Get the assigned permissions. + */ + public function permissions(): HasMany + { + return $this->hasMany(EntityPermission::class, 'entity_id') + ->whereColumn('entity_type', '=', 'entities.type'); + } + + /** + * Get View objects for this entity. + */ + public function views(): HasMany + { + return $this->hasMany(View::class, 'viewable_id') + ->whereColumn('viewable_type', '=', 'entities.type'); } } diff --git a/app/Entities/Queries/EntityQueries.php b/app/Entities/Queries/EntityQueries.php index c27cc61cc..91c6a4363 100644 --- a/app/Entities/Queries/EntityQueries.php +++ b/app/Entities/Queries/EntityQueries.php @@ -5,6 +5,7 @@ namespace BookStack\Entities\Queries; use BookStack\Entities\Models\Entity; use BookStack\Entities\Models\EntityTable; use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Query\Builder as QueryBuilder; use Illuminate\Database\Query\JoinClause; use Illuminate\Support\Facades\DB; use InvalidArgumentException; @@ -43,8 +44,14 @@ class EntityQueries public function visibleForList(): Builder { $rawDescriptionField = DB::raw('COALESCE(description, text) as description'); + $bookSlugSelect = function (QueryBuilder $query) { + return $query->select('slug')->from('entities as books') + ->whereColumn('books.id', '=', 'entities.book_id') + ->where('type', '=', 'book'); + }; + return EntityTable::query()->scopes('visible') - ->select(['id', 'type', 'name', 'slug', 'book_id', 'chapter_id', 'created_at', 'updated_at', 'draft', $rawDescriptionField]) + ->select(['id', 'type', 'name', 'slug', 'book_id', 'chapter_id', 'created_at', 'updated_at', 'draft', 'book_slug' => $bookSlugSelect, $rawDescriptionField]) ->leftJoin('entity_container_data', function (JoinClause $join) { $join->on('entity_container_data.entity_id', '=', 'entities.id') ->on('entity_container_data.entity_type', '=', 'entities.type'); diff --git a/app/Entities/Tools/EntityHydrator.php b/app/Entities/Tools/EntityHydrator.php index 7cabc7ecb..87e39d222 100644 --- a/app/Entities/Tools/EntityHydrator.php +++ b/app/Entities/Tools/EntityHydrator.php @@ -129,11 +129,11 @@ class EntityHydrator foreach ($entities as $entity) { if ($entity instanceof Page || $entity instanceof Chapter) { $key = 'book:' . $entity->getRawAttribute('book_id'); - $entity->setAttribute('book', $parentMap[$key] ?? null); + $entity->setRelation('book', $parentMap[$key] ?? null); } if ($entity instanceof Page) { $key = 'chapter:' . $entity->getRawAttribute('chapter_id'); - $entity->setAttribute('chapter', $parentMap[$key] ?? null); + $entity->setRelation('chapter', $parentMap[$key] ?? null); } } } diff --git a/app/Search/SearchRunner.php b/app/Search/SearchRunner.php index 0128e3804..bfb65cf0f 100644 --- a/app/Search/SearchRunner.php +++ b/app/Search/SearchRunner.php @@ -72,9 +72,9 @@ class SearchRunner $entityTypesToSearch = isset($filterMap['type']) ? explode('|', $filterMap['type']) : $entityTypes; $filteredTypes = array_intersect($entityTypesToSearch, $entityTypes); - $results = $this->buildQuery($opts, $filteredTypes)->where('book_id', '=', $bookId)->take(20)->get(); + $query = $this->buildQuery($opts, $filteredTypes)->where('book_id', '=', $bookId); - return $results->sortByDesc('score')->take(20); + return $this->getPageOfDataFromQuery($query, 1, 20)->sortByDesc('score'); } /** @@ -83,9 +83,9 @@ class SearchRunner public function searchChapter(int $chapterId, string $searchString): Collection { $opts = SearchOptions::fromString($searchString); - $pages = $this->buildQuery($opts, ['page'])->where('chapter_id', '=', $chapterId)->take(20)->get(); + $query = $this->buildQuery($opts, ['page'])->where('chapter_id', '=', $chapterId); - return $pages->sortByDesc('score'); + return $this->getPageOfDataFromQuery($query, 1, 20)->sortByDesc('score'); } /** @@ -120,7 +120,8 @@ class SearchRunner $filter = function (EloquentBuilder $query) use ($exact) { $inputTerm = str_replace('\\', '\\\\', $exact->value); $query->where('name', 'like', '%' . $inputTerm . '%') - ->orWhere('description', 'like', '%' . $inputTerm . '%'); + ->orWhere('description', 'like', '%' . $inputTerm . '%') + ->orWhere('text', 'like', '%' . $inputTerm . '%'); }; $exact->negated ? $entityQuery->whereNot($filter) : $entityQuery->where($filter); @@ -301,7 +302,7 @@ class SearchRunner $option->negated ? $query->whereDoesntHave('tags', $filter) : $query->whereHas('tags', $filter); } - protected function applyNegatableWhere(EloquentBuilder $query, bool $negated, string $column, string $operator, mixed $value): void + protected function applyNegatableWhere(EloquentBuilder $query, bool $negated, string|callable $column, string|null $operator, mixed $value): void { if ($negated) { $query->whereNot($column, $operator, $value); @@ -376,7 +377,10 @@ class SearchRunner protected function filterInBody(EloquentBuilder $query, string $input, bool $negated) { - $this->applyNegatableWhere($query, $negated, 'description', 'like', '%' . $input . '%'); + $this->applyNegatableWhere($query, $negated, function (EloquentBuilder $query) use ($input) { + $query->where('description', 'like', '%' . $input . '%') + ->orWhere('text', 'like', '%' . $input . '%'); + }, null, null); } protected function filterIsRestricted(EloquentBuilder $query, string $input, bool $negated) diff --git a/tests/Api/SearchApiTest.php b/tests/Api/SearchApiTest.php index 9da7900ca..517c5d8e4 100644 --- a/tests/Api/SearchApiTest.php +++ b/tests/Api/SearchApiTest.php @@ -113,6 +113,7 @@ class SearchApiTest extends TestCase $this->permissions->disableEntityInheritedPermissions($book); $resp = $this->getJson($this->baseEndpoint . '?query=superextrauniquevalue'); + $resp->assertOk(); $resp->assertJsonPath('data.0.id', $page->id); $resp->assertJsonMissingPath('data.0.book.name'); } diff --git a/tests/Search/EntitySearchTest.php b/tests/Search/EntitySearchTest.php index d3d859986..ca900c2ed 100644 --- a/tests/Search/EntitySearchTest.php +++ b/tests/Search/EntitySearchTest.php @@ -27,6 +27,12 @@ class EntitySearchTest extends TestCase $search->assertSeeText($shelf->name, true); } + public function test_search_shows_pagination() + { + $search = $this->asEditor()->get('/search?term=a'); + $this->withHtml($search)->assertLinkExists('/search?term=a&page=2', '2'); + } + public function test_invalid_page_search() { $resp = $this->asEditor()->get('/search?term=' . urlencode('

test

'));