import os import time import gradio as gr from modules import ui_symbols, ui_components from installer import install, log class Page(): def __init__(self, fn, full: bool = True): self.fn = fn self.title = '' self.size = 0 self.mtime = 0 self.h1 = [] self.h2 = [] self.h3 = [] self.lines = [] self.read(full=full) def read(self, full: bool = True): try: self.title = ' ' + os.path.basename(self.fn).replace('.md', '').replace('-', ' ') + ' ' self.mtime = time.localtime(os.path.getmtime(self.fn)) with open(self.fn, 'r', encoding='utf-8') as f: content = f.read() self.size = len(content) self.lines = [line.strip().lower() + ' ' for line in content.splitlines() if len(line)>1] self.h1 = [line[1:] for line in self.lines if line.startswith('# ')] self.h2 = [line[2:] for line in self.lines if line.startswith('## ')] self.h3 = [line[3:] for line in self.lines if line.startswith('### ')] if not full: self.lines.clear() except Exception as e: log.error(f'Search docs: page="{self.fn}" {e}') def search(self, text): if not text or len(text) < 2: return [] text = text.lower() if text.strip() == self.title.lower().strip(): return 1.0 if self.title.lower().startswith(f'{text} '): return 0.99 if f' {text} ' in self.title.lower(): return 0.98 if f' {text}' in self.title.lower(): return 0.97 if any(f' {text} ' in h for h in self.h1): return 0.89 if any(f' {text}' in h for h in self.h1): return 0.88 if any(f' {text} ' in h for h in self.h2): return 0.79 if any(f' {text}' in h for h in self.h2): return 0.78 if any(f' {text} ' in h for h in self.h3): return 0.69 if any(f' {text}' in h for h in self.h3): return 0.68 if f'{text}' in self.title.lower(): return 0.59 if any(f'{text}' in h for h in self.h1): return 0.58 if any(f'{text}' in h for h in self.h2): return 0.57 if any(f'{text}' in h for h in self.h3): return 0.56 if any(text in line for line in self.lines): return 0.50 return 0.0 def get(self): if self.fn is None or not os.path.exists(self.fn): log.error(f'Search docs: page="{self.fn}" does not exist') return f'page="{self.fn}" does not exist' try: with open(self.fn, 'r', encoding='utf-8') as f: content = f.read() return content except Exception as e: log.error(f'Search docs: page="{self.fn}" {e}') return '' def __str__(self): return f'Page(title="{self.title.strip()}" fn="{self.fn}" mtime={self.mtime} h1={[h.strip() for h in self.h1]} h2={len(self.h2)} h3={len(self.h3)} lines={len(self.lines)} size={self.size})' class Pages(): def __init__(self): self.time = time.time() self.size = 0 self.full = None self.pages: list[Page] = [] def build(self, full: bool = True): self.pages.clear() self.full = full with os.scandir('wiki') as entries: for entry in entries: if entry.is_file() and entry.name.endswith('.md'): page = Page(entry.path, full=full) self.pages.append(page) self.size = sum(page.size for page in self.pages) def search(self, text: str, topk: int = 10, full: bool = True) -> list[Page]: if not text or len(text) < 2: return [] if len(self.pages) == 0: self.build(full=full) try: text = text.lower() scores = [page.search(text) for page in self.pages] mtimes = [page.mtime for page in self.pages] found = sorted(zip(scores, mtimes, self.pages), key=lambda x: (x[0], x[1]), reverse=True) found = [item for item in found if item[0] > 0] return [(item[0], item[2]) for item in found][:topk] except Exception as e: log.error(f'Search docs: text="{text}" {e}') return [] def get(self, title: str) -> Page: if len(self.pages) == 0: self.build(full=self.full) for page in self.pages: if page.title.lower().strip() == title.lower().strip(): return page return Page('') index = Pages() def get_docs_page(page_title: str) -> str: if len(index.pages) == 0: index.build(full=True) page = index.get(page_title) log.debug(f'Search docs: title="{page_title}" {page}') content = page.get() return content def search_html(pages: list[Page]) -> str: html = '' for score, page in pages: if score > 0.0: html += f'''
{page.title.strip()}
Heading | {' | '.join([h.strip() for h in page.h1])}
Topics | {' | '.join([h.strip() for h in page.h2])}
''' return html def search_docs(search_term): topk = 10 full = True t0 = time.time() results = index.search(search_term, topk=topk, full=full) t1 = time.time() log.debug(f'Search results: search="{search_term}" topk={topk}, full={full} pages={len(results)} size={index.size} time={t1-t0:.3f}') for score, page in results: log.trace(f'Search results: score={score:.2f} {page}') html = search_html(results) return html def get_github_page(page): try: with open(os.path.join('wiki', f'{page}.md'), 'r', encoding='utf-8') as f: content = f.read() log.debug(f'Search wiki: page="{page}" size={len(content)}') except Exception as e: log.error(f'Search wiki: page="{page}" {e}') content = f'Error: {e}' return content def search_github(search_term): import requests from urllib.parse import quote install('beautifulsoup4') from bs4 import BeautifulSoup url = f'https://github.com/search?q=repo%3Avladmandic%2Fsdnext+{quote(search_term)}&type=wikis' res = requests.get(url, timeout=10) pages = [] if res.status_code == 200: html = res.content soup = BeautifulSoup(html, 'html.parser') # remove header links tags = soup.find_all(attrs={"data-hovercard-url": "/vladmandic/sdnext/hovercard"}) for tag in tags: tag.extract() # replace relative links with full links tags = soup.find_all('a') for tag in tags: if tag.has_attr('href'): if tag['href'].startswith('/vladmandic/sdnext/wiki/'): page = tag['href'].replace('/vladmandic/sdnext/wiki/', '') tag.name = 'div' tag['class'] = 'github-page' tag['onclick'] = f'clickGitHubWikiPage("{page}")' pages.append(page) elif tag['href'].startswith('/'): tag['href'] = 'https://github.com' + tag['href'] # find result only result = soup.find(attrs={"data-testid": "results-list"}) if result is None: return 'No results found' html = str(result) else: html = f'Error: {res.status_code}' log.debug(f'Search wiki: code={res.status_code} text="{search_term}" pages={pages}') return html def create_ui_logs(): def get_changelog(): with open('CHANGELOG.md', 'r', encoding='utf-8') as f: content = f.read() content = content.replace('# Change Log for SD.Next', ' ') return content with gr.Column(): get_changelog_btn = gr.Button(value='Get Changelog', elem_id="get_changelog") with gr.Column(): _changelog_search = gr.Textbox(label="Search Changelog", elem_id="changelog_search", elem_classes="docs-search") _changelog_result = gr.HTML(elem_id="changelog_result") changelog_markdown = gr.Markdown('', elem_id="changelog_markdown") get_changelog_btn.click(fn=get_changelog, outputs=[changelog_markdown], show_progress='full') def create_ui_github(): with gr.Row(): github_search = gr.Textbox(label="Search GitHub Wiki Pages", elem_id="github_search", elem_classes="docs-search") github_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="github_btn_search") with gr.Row(): github_result = gr.HTML(elem_id="github_result", value='', elem_classes="github-result") with gr.Row(): github_md_btn = gr.Button(value='html2md', elem_id="github_md_btn", visible=False) github_md = gr.Markdown(elem_id="github_md", value='', elem_classes="github-md") github_search.submit(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress='full') github_search_btn.click(fn=search_github, inputs=[github_search], outputs=[github_result], show_progress='full') github_md_btn.click(fn=get_github_page, _js='getGitHubWikiPage', inputs=[github_search], outputs=[github_md], show_progress='full') def create_ui_docs(): with gr.Row(): docs_search = gr.Textbox(label="Search Docs", elem_id="github_search", elem_classes="docs-search") docs_search_btn = ui_components.ToolButton(value=ui_symbols.search, elem_id="docs_btn_search") with gr.Row(): docs_result = gr.HTML(elem_id="docs_result", value='', elem_classes="docs-result") with gr.Row(): docs_md_btn = gr.Button(value='html2md', elem_id="docs_md_btn", visible=False) docs_md = gr.Markdown(elem_id="docs_md", value='', elem_classes="docs-md") docs_search.submit(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress='hidden') docs_search.change(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress='hidden') docs_search_btn.click(fn=search_docs, inputs=[docs_search], outputs=[docs_result], show_progress='hidden') docs_md_btn.click(fn=get_docs_page, _js='getDocsPage', inputs=[docs_search], outputs=[docs_md], show_progress='hidden') def create_ui(): log.debug('UI initialize: tab=info') with gr.Tabs(elem_id="tabs_info"): with gr.TabItem("Docs", id="docs", elem_id="system_tab_docs"): create_ui_docs() with gr.TabItem("Wiki", id="wiki", elem_id="system_tab_wiki"): create_ui_github() with gr.TabItem("Change log", id="change_log", elem_id="system_tab_changelog"): create_ui_logs()