revert use of gitpython hijack

Allow activation of Generate Forever during generation

.eslintignore

@@ -0,0 +1,4 @@
+extensions
+extensions-disabled
+repositories
+venv

.eslintrc.js

@@ -0,0 +1,91 @@
+/* global module */
+module.exports = {
+    env: {
+        browser: true,
+        es2021: true,
+    },
+    extends: "eslint:recommended",
+    parserOptions: {
+        ecmaVersion: "latest",
+    },
+    rules: {
+        "arrow-spacing": "error",
+        "block-spacing": "error",
+        "brace-style": "error",
+        "comma-dangle": ["error", "only-multiline"],
+        "comma-spacing": "error",
+        "comma-style": ["error", "last"],
+        "curly": ["error", "multi-line", "consistent"],
+        "eol-last": "error",
+        "func-call-spacing": "error",
+        "function-call-argument-newline": ["error", "consistent"],
+        "function-paren-newline": ["error", "consistent"],
+        "indent": ["error", 4],
+        "key-spacing": "error",
+        "keyword-spacing": "error",
+        "linebreak-style": ["error", "unix"],
+        "no-extra-semi": "error",
+        "no-mixed-spaces-and-tabs": "error",
+        "no-multi-spaces": "error",
+        "no-redeclare": ["error", {builtinGlobals: false}],
+        "no-trailing-spaces": "error",
+        "no-unused-vars": "off",
+        "no-whitespace-before-property": "error",
+        "object-curly-newline": ["error", {consistent: true, multiline: true}],
+        "object-curly-spacing": ["error", "never"],
+        "operator-linebreak": ["error", "after"],
+        "quote-props": ["error", "consistent-as-needed"],
+        "semi": ["error", "always"],
+        "semi-spacing": "error",
+        "semi-style": ["error", "last"],
+        "space-before-blocks": "error",
+        "space-before-function-paren": ["error", "never"],
+        "space-in-parens": ["error", "never"],
+        "space-infix-ops": "error",
+        "space-unary-ops": "error",
+        "switch-colon-spacing": "error",
+        "template-curly-spacing": ["error", "never"],
+        "unicode-bom": "error",
+    },
+    globals: {
+        //script.js
+        gradioApp: "readonly",
+        executeCallbacks: "readonly",
+        onAfterUiUpdate: "readonly",
+        onOptionsChanged: "readonly",
+        onUiLoaded: "readonly",
+        onUiUpdate: "readonly",
+        uiCurrentTab: "writable",
+        uiElementInSight: "readonly",
+        uiElementIsVisible: "readonly",
+        //ui.js
+        opts: "writable",
+        all_gallery_buttons: "readonly",
+        selected_gallery_button: "readonly",
+        selected_gallery_index: "readonly",
+        switch_to_txt2img: "readonly",
+        switch_to_img2img_tab: "readonly",
+        switch_to_img2img: "readonly",
+        switch_to_sketch: "readonly",
+        switch_to_inpaint: "readonly",
+        switch_to_inpaint_sketch: "readonly",
+        switch_to_extras: "readonly",
+        get_tab_index: "readonly",
+        create_submit_args: "readonly",
+        restart_reload: "readonly",
+        updateInput: "readonly",
+        //extraNetworks.js
+        requestGet: "readonly",
+        popup: "readonly",
+        // from python
+        localization: "readonly",
+        // progrssbar.js
+        randomId: "readonly",
+        requestProgress: "readonly",
+        // imageviewer.js
+        modalPrevImage: "readonly",
+        modalNextImage: "readonly",
+        // token-counters.js
+        setupTokenCounters: "readonly",
+    }
+};

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Apply ESlint
+9c54b78d9dde5601e916f308d9a9d6953ec39430

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -37,20 +37,29 @@ body:
     id: what-should
     attributes:
       label: What should have happened?
-      description: tell what you think the normal behavior should be
+      description: Tell what you think the normal behavior should be
     validations:
       required: true
   - type: input
     id: commit
     attributes:
-      label: Commit where the problem happens
-      description: Which commit are you running ? (Do not write *Latest version/repo/commit*, as this means nothing and will have changed by the time we read your issue. Rather, copy the **Commit hash** shown in the cmd/terminal when you launch the UI)
+      label: Version or Commit where the problem happens
+      description: "Which webui version or commit are you running ? (Do not write *Latest Version/repo/commit*, as this means nothing and will have changed by the time we read your issue. Rather, copy the **Version: v1.2.3** link at the bottom of the UI, or from the cmd/terminal if you can't launch it.)"
     validations:
       required: true
+  - type: dropdown
+    id: py-version
+    attributes:
+      label: What Python version are you running on ?
+      multiple: false
+      options:
+        - Python 3.10.x
+        - Python 3.11.x (above, no supported yet)
+        - Python 3.9.x (below, no recommended)
   - type: dropdown
     id: platforms
     attributes:
-      label: What platforms do you use to access UI ?
+      label: What platforms do you use to access the UI ?
       multiple: true
       options:
         - Windows
@@ -59,6 +68,35 @@ body:
         - iOS
         - Android
         - Other/Cloud
+  - type: dropdown
+    id: device
+    attributes:
+        label: What device are you running WebUI on?
+        multiple: true
+        options:
+        - Nvidia GPUs (RTX 20 above)
+        - Nvidia GPUs (GTX 16 below)
+        - AMD GPUs (RX 6000 above)
+        - AMD GPUs (RX 5000 below)
+        - CPU
+        - Other GPUs
+  - type: dropdown
+    id: cross_attention_opt
+    attributes:
+      label: Cross attention optimization
+      description: What cross attention optimization are you using, Settings -> Optimizations -> Cross attention optimization
+      multiple: false
+      options:
+        - Automatic
+        - xformers
+        - sdp-no-mem
+        - sdp
+        - Doggettx
+        - V1 
+        - InvokeAI
+        - "None "
+    validations:
+      required: true
   - type: dropdown
     id: browsers
     attributes:
@@ -74,10 +112,27 @@ body:
     id: cmdargs
     attributes:
       label: Command Line Arguments
-      description: Are you using any launching parameters/command line arguments (modified webui-user.py) ? If yes, please write them below
+      description: Are you using any launching parameters/command line arguments (modified webui-user .bat/.sh) ? If yes, please write them below. Write "No" otherwise.
       render: Shell
+    validations:
+      required: true
+  - type: textarea
+    id: extensions
+    attributes:
+      label: List of extensions
+      description: Are you using any extensions other than built-ins? If yes, provide a list, you can copy it at "Extensions" tab. Write "No" otherwise.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Console logs
+      description: Please provide **full** cmd/terminal logs from the moment you started UI to the end of it, after your bug happened. If it's very long, provide a link to pastebin or similar service.
+      render: Shell
+    validations:
+      required: true
   - type: textarea
     id: misc
     attributes:
-      label: Additional information, context and logs
-      description: Please provide us with any relevant additional info, context or log output.
+      label: Additional information
+      description: Please provide us with any relevant additional info or context.

.github/pull_request_template.md

@@ -1,28 +1,15 @@
-# Please read the [contributing wiki page](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Contributing) before submitting a pull request!
+## Description
 
-If you have a large change, pay special attention to this paragraph:
+* a simple description of what you're trying to accomplish
+* a summary of changes in code
+* which issues it fixes, if any
 
-> Before making changes, if you think that your feature will result in more than 100 lines changing, find me and talk to me about the feature you are proposing. It pains me to reject the hard work someone else did, but I won't add everything to the repo, and it's better if the rejection happens before you have to waste time working on the feature.
+## Screenshots/videos:
 
-Otherwise, after making sure you're following the rules described in wiki page, remove this section and continue on.
 
-**Describe what this pull request is trying to achieve.**
+## Checklist:
 
-A clear and concise description of what you're trying to accomplish with this, so your intent doesn't have to be extracted from your code.
-
-**Additional notes and description of your changes**
-
-More technical discussion about your changes go here, plus anything that a maintainer might have to specifically take a look at, or be wary of.
-
-**Environment this was tested in**
-
-List the environment you have developed / tested this on. As per the contributing page, changes should be able to work on Windows out of the box.
- - OS: [e.g. Windows, Linux]
- - Browser: [e.g. chrome, safari]
- - Graphics card: [e.g. NVIDIA RTX 2080 8GB, AMD RX 6600 8GB]
-
-**Screenshots or videos of your changes**
-
-If applicable, screenshots or a video showing off your changes. If it edits an existing UI, it should ideally contain a comparison of what used to be there, before your changes were made.
-
-This is **required** for anything that touches the user interface.
+- [ ] I have read [contributing wiki page](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Contributing)
+- [ ] I have performed a self-review of my own code
+- [ ] My code follows the [style guidelines](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Contributing#code-style)
+- [ ] My code passes [tests](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Tests)

.github/workflows/on_pull_request.yaml

@@ -1,39 +1,34 @@
-# See https://github.com/actions/starter-workflows/blob/1067f16ad8a1eac328834e4b0ae24f7d206f810d/ci/pylint.yml for original reference file
 name: Run Linting/Formatting on Pull Requests
 
 on:
   - push
   - pull_request
-  # See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onpull_requestpull_request_targetbranchesbranches-ignore for syntax docs
-  # if you want to filter out branches, delete the `- pull_request` and uncomment these lines :
-  # pull_request:  
-  #  branches:
-  #    - master
-  #  branches-ignore:
-  #    - development
 
 jobs:
-  lint:
+  lint-python:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+      - uses: actions/setup-python@v4
         with:
-          python-version: 3.10.6
-          cache: pip
-          cache-dependency-path: |
-            **/requirements*txt
-      - name: Install PyLint
-        run: | 
-          python -m pip install --upgrade pip
-          pip install pylint
-      # This lets PyLint check to see if it can resolve imports
-      - name: Install dependencies
-        run: |
-          export COMMANDLINE_ARGS="--skip-torch-cuda-test --exit"
-          python launch.py
-      - name: Analysing the code with pylint
-        run: |
-          pylint $(git ls-files '*.py')
+          python-version: 3.11
+          # NB: there's no cache: pip here since we're not installing anything
+          #     from the requirements.txt file(s) in the repository; it's faster
+          #     not to have GHA download an (at the time of writing) 4 GB cache
+          #     of PyTorch and other dependencies.
+      - name: Install Ruff
+        run: pip install ruff==0.0.265
+      - name: Run Ruff
+        run: ruff .
+  lint-js:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - run: npm i --ci
+      - run: npm run lint

.github/workflows/run_tests.yaml

@@ -17,13 +17,54 @@ jobs:
           cache: pip
           cache-dependency-path: |
             **/requirements*txt
+            launch.py
+      - name: Install test dependencies
+        run: pip install wait-for-it -r requirements-test.txt
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+      - name: Setup environment
+        run: python launch.py --skip-torch-cuda-test --exit
+        env:
+          PIP_DISABLE_PIP_VERSION_CHECK: "1"
+          PIP_PROGRESS_BAR: "off"
+          TORCH_INDEX_URL: https://download.pytorch.org/whl/cpu
+          WEBUI_LAUNCH_LIVE_OUTPUT: "1"
+          PYTHONUNBUFFERED: "1"
+      - name: Start test server
+        run: >
+          python -m coverage run
+          --data-file=.coverage.server
+          launch.py
+          --skip-prepare-environment
+          --skip-torch-cuda-test
+          --test-server
+          --no-half
+          --disable-opt-split-attention
+          --use-cpu all
+          --add-stop-route
+          2>&1 | tee output.txt &
       - name: Run tests
-        run: python launch.py --tests --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
-      - name: Upload main app stdout-stderr
+        run: |
+          wait-for-it --service 127.0.0.1:7860 -t 600
+          python -m pytest -vv --junitxml=test/results.xml --cov . --cov-report=xml --verify-base-url test
+      - name: Kill test server
+        if: always()
+        run: curl -vv -XPOST http://127.0.0.1:7860/_stop && sleep 10
+      - name: Show coverage
+        run: |
+          python -m coverage combine .coverage*
+          python -m coverage report -i
+          python -m coverage html -i
+      - name: Upload main app output
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: stdout-stderr
-          path: |
-            test/stdout.txt
-            test/stderr.txt
+          name: output
+          path: output.txt
+      - name: Upload coverage HTML
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: htmlcov
+          path: htmlcov

.gitignore

@@ -32,4 +32,8 @@ notification.mp3
 /extensions
 /test/stdout.txt
 /test/stderr.txt
-/cache.json
+/cache.json*
+/config_states/
+/node_modules
+/package-lock.json
+/.coverage*

CHANGELOG.md

@@ -0,0 +1,257 @@
+## 1.4.0
+
+### Features:
+ * zoom controls for inpainting
+ * run basic torch calculation at startup in parallel to reduce the performance impact of first generation
+ * option to pad prompt/neg prompt to be same length
+ * remove taming_transformers dependency
+ * custom k-diffusion scheduler settings
+ * add an option to show selected settings in main txt2img/img2img UI
+ * sysinfo tab in settings
+ * infer styles from prompts when pasting params into the UI
+ * an option to control the behavior of the above
+
+### Minor:
+ * bump Gradio to 3.32.0
+ * bump xformers to 0.0.20
+ * Add option to disable token counters
+ * tooltip fixes & optimizations
+ * make it possible to configure filename for the zip download
+ * `[vae_filename]` pattern for filenames
+ * Revert discarding penultimate sigma for DPM-Solver++(2M) SDE
+ * change UI reorder setting to multiselect
+ * read version info form CHANGELOG.md if git version info is not available
+ * link footer API to Wiki when API is not active
+ * persistent conds cache (opt-in optimization)
+ 
+### Extensions:
+ * After installing extensions, webui properly restarts the process rather than reloads the UI 
+ * Added VAE listing to web API. Via: /sdapi/v1/sd-vae
+ * custom unet support
+ * Add onAfterUiUpdate callback
+ * refactor EmbeddingDatabase.register_embedding() to allow unregistering
+ * add before_process callback for scripts
+ * add ability for alwayson scripts to specify section and let user reorder those sections
+ 
+### Bug Fixes:
+ * Fix dragging text to prompt
+ * fix incorrect quoting for infotext values with colon in them
+ * fix "hires. fix" prompt sharing same labels with txt2img_prompt
+ * Fix s_min_uncond default type int
+ * Fix for #10643 (Inpainting mask sometimes not working)
+ * fix bad styling for thumbs view in extra networks #10639
+ * fix for empty list of optimizations #10605
+ * small fixes to prepare_tcmalloc for Debian/Ubuntu compatibility
+ * fix --ui-debug-mode exit
+ * patch GitPython to not use leaky persistent processes
+ * fix duplicate Cross attention optimization after UI reload
+ * torch.cuda.is_available() check for SdOptimizationXformers
+ * fix hires fix using wrong conds in second pass if using Loras.
+ * handle exception when parsing generation parameters from png info
+ * fix upcast attention dtype error
+ * forcing Torch Version to 1.13.1 for RX 5000 series GPUs
+ * split mask blur into X and Y components, patch Outpainting MK2 accordingly
+ * don't die when a LoRA is a broken symlink
+ * allow activation of Generate Forever during generation
+
+
+## 1.3.2
+
+### Bug Fixes:
+ * fix files served out of tmp directory even if they are saved to disk
+ * fix postprocessing overwriting parameters
+
+## 1.3.1
+
+### Features:
+ * revert default cross attention optimization to Doggettx
+
+### Bug Fixes:
+ * fix bug: LoRA don't apply on dropdown list sd_lora
+ * fix png info always added even if setting is not enabled
+ * fix some fields not applying in xyz plot
+ * fix "hires. fix" prompt sharing same labels with txt2img_prompt
+ * fix lora hashes not being added properly to infotex if there is only one lora
+ * fix --use-cpu failing to work properly at startup
+ * make --disable-opt-split-attention command line option work again
+
+## 1.3.0
+
+### Features:
+ * add UI to edit defaults
+ * token merging (via dbolya/tomesd)
+ * settings tab rework: add a lot of additional explanations and links
+ * load extensions' Git metadata in parallel to loading the main program to save a ton of time during startup
+ * update extensions table: show branch, show date in separate column, and show version from tags if available
+ * TAESD - another option for cheap live previews
+ * allow choosing sampler and prompts for second pass of hires fix - hidden by default, enabled in settings
+ * calculate hashes for Lora
+ * add lora hashes to infotext
+ * when pasting infotext, use infotext's lora hashes to find local loras for `<lora:xxx:1>` entries whose hashes match loras the user has
+ * select cross attention optimization from UI
+
+### Minor:
+ * bump Gradio to 3.31.0
+ * bump PyTorch to 2.0.1 for macOS and Linux AMD
+ * allow setting defaults for elements in extensions' tabs
+ * allow selecting file type for live previews
+ * show "Loading..." for extra networks when displaying for the first time
+ * suppress ENSD infotext for samplers that don't use it
+ * clientside optimizations
+ * add options to show/hide hidden files and dirs in extra networks, and to not list models/files in hidden directories
+ * allow whitespace in styles.csv
+ * add option to reorder tabs
+ * move some functionality (swap resolution and set seed to -1) to client
+ * option to specify editor height for img2img
+ * button to copy image resolution into img2img width/height sliders
+ * switch from pyngrok to ngrok-py
+ * lazy-load images in extra networks UI
+ * set "Navigate image viewer with gamepad" option to false by default, by request
+ * change upscalers to download models into user-specified directory (from commandline args) rather than the default models/<...>
+ * allow hiding buttons in ui-config.json
+
+### Extensions:
+ * add /sdapi/v1/script-info api
+ * use Ruff to lint Python code
+ * use ESlint to lint Javascript code
+ * add/modify CFG callbacks for Self-Attention Guidance extension
+ * add command and endpoint for graceful server stopping
+ * add some locals (prompts/seeds/etc) from processing function into the Processing class as fields
+ * rework quoting for infotext items that have commas in them to use JSON (should be backwards compatible except for cases where it didn't work previously)
+ * add /sdapi/v1/refresh-loras api checkpoint post request
+ * tests overhaul
+
+### Bug Fixes:
+ * fix an issue preventing the program from starting if the user specifies a bad Gradio theme
+ * fix broken prompts from file script
+ * fix symlink scanning for extra networks
+ * fix --data-dir ignored when launching via webui-user.bat COMMANDLINE_ARGS
+ * allow web UI to be ran fully offline
+ * fix inability to run with --freeze-settings
+ * fix inability to merge checkpoint without adding metadata
+ * fix extra networks' save preview image not adding infotext for jpeg/webm
+ * remove blinking effect from text in hires fix and scale resolution preview
+ * make links to `http://<...>.git` extensions work in the extension tab
+ * fix bug with webui hanging at startup due to hanging git process
+
+
+## 1.2.1
+
+### Features:
+ * add an option to always refer to LoRA by filenames
+
+### Bug Fixes:
+ * never refer to LoRA by an alias if multiple LoRAs have same alias or the alias is called none
+ * fix upscalers disappearing after the user reloads UI
+ * allow bf16 in safe unpickler (resolves problems with loading some LoRAs)
+ * allow web UI to be ran fully offline
+ * fix localizations not working
+ * fix error for LoRAs: `'LatentDiffusion' object has no attribute 'lora_layer_mapping'`
+
+## 1.2.0
+
+### Features:
+ * do not wait for Stable Diffusion model to load at startup
+ * add filename patterns: `[denoising]`
+ * directory hiding for extra networks: dirs starting with `.` will hide their cards on extra network tabs unless specifically searched for
+ * LoRA: for the `<...>` text in prompt, use name of LoRA that is in the metdata of the file, if present, instead of filename (both can be used to activate LoRA)
+ * LoRA: read infotext params from kohya-ss's extension parameters if they are present and if his extension is not active
+ * LoRA: fix some LoRAs not working (ones that have 3x3 convolution layer)
+ * LoRA: add an option to use old method of applying LoRAs (producing same results as with kohya-ss)
+ * add version to infotext, footer and console output when starting
+ * add links to wiki for filename pattern settings
+ * add extended info for quicksettings setting and use multiselect input instead of a text field
+
+### Minor:
+ * bump Gradio to 3.29.0
+ * bump PyTorch to 2.0.1
+ * `--subpath` option for gradio for use with reverse proxy
+ * Linux/macOS: use existing virtualenv if already active (the VIRTUAL_ENV environment variable)
+ * do not apply localizations if there are none (possible frontend optimization)
+ * add extra `None` option for VAE in XYZ plot
+ * print error to console when batch processing in img2img fails
+ * create HTML for extra network pages only on demand
+ * allow directories starting with `.` to still list their models for LoRA, checkpoints, etc
+ * put infotext options into their own category in settings tab
+ * do not show licenses page when user selects Show all pages in settings
+
+### Extensions:
+ * tooltip localization support
+ * add API method to get LoRA models with prompt
+
+### Bug Fixes:
+ * re-add `/docs` endpoint
+ * fix gamepad navigation
+ * make the lightbox fullscreen image function properly
+ * fix squished thumbnails in extras tab
+ * keep "search" filter for extra networks when user refreshes the tab (previously it showed everthing after you refreshed)
+ * fix webui showing the same image if you configure the generation to always save results into same file
+ * fix bug with upscalers not working properly
+ * fix MPS on PyTorch 2.0.1, Intel Macs
+ * make it so that custom context menu from contextMenu.js only disappears after user's click, ignoring non-user click events
+ * prevent Reload UI button/link from reloading the page when it's not yet ready
+ * fix prompts from file script failing to read contents from a drag/drop file
+
+
+## 1.1.1
+### Bug Fixes:
+ * fix an error that prevents running webui on PyTorch<2.0 without --disable-safe-unpickle
+
+## 1.1.0
+### Features:
+ * switch to PyTorch 2.0.0 (except for AMD GPUs)
+ * visual improvements to custom code scripts
+ * add filename patterns: `[clip_skip]`, `[hasprompt<>]`, `[batch_number]`, `[generation_number]`
+ * add support for saving init images in img2img, and record their hashes in infotext for reproducability
+ * automatically select current word when adjusting weight with ctrl+up/down
+ * add dropdowns for X/Y/Z plot
+ * add setting: Stable Diffusion/Random number generator source: makes it possible to make images generated from a given manual seed consistent across different GPUs
+ * support Gradio's theme API
+ * use TCMalloc on Linux by default; possible fix for memory leaks
+ * add optimization option to remove negative conditioning at low sigma values #9177
+ * embed model merge metadata in .safetensors file
+ * extension settings backup/restore feature #9169
+ * add "resize by" and "resize to" tabs to img2img
+ * add option "keep original size" to textual inversion images preprocess
+ * image viewer scrolling via analog stick
+ * button to restore the progress from session lost / tab reload
+
+### Minor:
+ * bump Gradio to 3.28.1
+ * change "scale to" to sliders in Extras tab
+ * add labels to tool buttons to make it possible to hide them
+ * add tiled inference support for ScuNET
+ * add branch support for extension installation
+ * change Linux installation script to install into current directory rather than `/home/username`
+ * sort textual inversion embeddings by name (case-insensitive)
+ * allow styles.csv to be symlinked or mounted in docker
+ * remove the "do not add watermark to images" option
+ * make selected tab configurable with UI config
+ * make the extra networks UI fixed height and scrollable
+ * add `disable_tls_verify` arg for use with self-signed certs
+
+### Extensions:
+ * add reload callback
+ * add `is_hr_pass` field for processing
+
+### Bug Fixes:
+ * fix broken batch image processing on 'Extras/Batch Process' tab
+ * add "None" option to extra networks dropdowns
+ * fix FileExistsError for CLIP Interrogator
+ * fix /sdapi/v1/txt2img endpoint not working on Linux #9319
+ * fix disappearing live previews and progressbar during slow tasks
+ * fix fullscreen image view not working properly in some cases
+ * prevent alwayson_scripts args param resizing script_arg list when they are inserted in it
+ * fix prompt schedule for second order samplers
+ * fix image mask/composite for weird resolutions #9628
+ * use correct images for previews when using AND (see #9491)
+ * one broken image in img2img batch won't stop all processing
+ * fix image orientation bug in train/preprocess
+ * fix Ngrok recreating tunnels every reload
+ * fix `--realesrgan-models-path` and `--ldsr-models-path` not working
+ * fix `--skip-install` not working
+ * use SAMPLE file format in Outpainting Mk2 & Poorman
+ * do not fail all LoRAs if some have failed to load when making a picture
+
+## 1.0.0
+  * everything

README.md

@@ -13,11 +13,11 @@ A browser interface based on Gradio library for Stable Diffusion.
 - Prompt Matrix
 - Stable Diffusion Upscale
 - Attention, specify parts of text that the model should pay more attention to
-    - a man in a ((tuxedo)) - will pay more attention to tuxedo
-    - a man in a (tuxedo:1.21) - alternative syntax
-    - select text and press ctrl+up or ctrl+down to automatically adjust attention to selected text (code contributed by anonymous user)
+    - a man in a `((tuxedo))` - will pay more attention to tuxedo
+    - a man in a `(tuxedo:1.21)` - alternative syntax
+    - select text and press `Ctrl+Up` or `Ctrl+Down` (or `Command+Up` or `Command+Down` if you're on a MacOS) to automatically adjust attention to selected text (code contributed by anonymous user)
 - Loopback, run img2img processing multiple times
-- X/Y plot, a way to draw a 2 dimensional plot of images with different parameters
+- X/Y/Z plot, a way to draw a 3 dimensional plot of images with different parameters
 - Textual Inversion
     - have as many embeddings as you want and use any names you like for them
     - use multiple embeddings with different numbers of vectors per token
@@ -28,7 +28,7 @@ A browser interface based on Gradio library for Stable Diffusion.
     - CodeFormer, face restoration tool as an alternative to GFPGAN
     - RealESRGAN, neural network upscaler
     - ESRGAN, neural network upscaler with a lot of third party models
-    - SwinIR and Swin2SR([see here](https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/2092)), neural network upscalers
+    - SwinIR and Swin2SR ([see here](https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/2092)), neural network upscalers
     - LDSR, Latent diffusion super resolution upscaling
 - Resizing aspect ratio options
 - Sampling method selection
@@ -46,7 +46,7 @@ A browser interface based on Gradio library for Stable Diffusion.
      - drag and drop an image/text-parameters to promptbox
 - Read Generation Parameters Button, loads parameters in promptbox to UI
 - Settings page
-- Running arbitrary python code from UI (must run with --allow-code to enable)
+- Running arbitrary python code from UI (must run with `--allow-code` to enable)
 - Mouseover hints for most UI elements
 - Possible to change defaults/mix/max/step values for UI elements via text config
 - Tiling support, a checkbox to create images that can be tiled like textures
@@ -69,7 +69,7 @@ A browser interface based on Gradio library for Stable Diffusion.
      - also supports weights for prompts: `a cat :1.2 AND a dog AND a penguin :2.2`
 - No token limit for prompts (original stable diffusion lets you use up to 75 tokens)
 - DeepDanbooru integration, creates danbooru style tags for anime prompts
-- [xformers](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Xformers), major speed increase for select cards: (add --xformers to commandline args)
+- [xformers](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Xformers), major speed increase for select cards: (add `--xformers` to commandline args)
 - via extension: [History tab](https://github.com/yfszzx/stable-diffusion-webui-images-browser): view, direct and delete images conveniently within the UI
 - Generate forever option
 - Training tab
@@ -78,11 +78,11 @@ A browser interface based on Gradio library for Stable Diffusion.
 - Clip skip
 - Hypernetworks
 - Loras (same as Hypernetworks but more pretty)
-- A sparate UI where you can choose, with preview, which embeddings, hypernetworks or Loras to add to your prompt. 
+- A sparate UI where you can choose, with preview, which embeddings, hypernetworks or Loras to add to your prompt 
 - Can select to load a different VAE from settings screen
 - Estimated completion time in progress bar
 - API
-- Support for dedicated [inpainting model](https://github.com/runwayml/stable-diffusion#inpainting-with-stable-diffusion) by RunwayML.
+- Support for dedicated [inpainting model](https://github.com/runwayml/stable-diffusion#inpainting-with-stable-diffusion) by RunwayML
 - via extension: [Aesthetic Gradients](https://github.com/AUTOMATIC1111/stable-diffusion-webui-aesthetic-gradients), a way to generate images with a specific aesthetic by using clip images embeds (implementation of [https://github.com/vicgalle/stable-diffusion-aesthetic-gradients](https://github.com/vicgalle/stable-diffusion-aesthetic-gradients))
 - [Stable Diffusion 2.0](https://github.com/Stability-AI/stablediffusion) support - see [wiki](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#stable-diffusion-20) for instructions
 - [Alt-Diffusion](https://arxiv.org/abs/2211.06679) support - see [wiki](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#alt-diffusion) for instructions
@@ -91,7 +91,6 @@ A browser interface based on Gradio library for Stable Diffusion.
 - Eased resolution restriction: generated image's domension must be a multiple of 8 rather than 64
 - Now with a license!
 - Reorder elements in the UI from settings screen
-- 
 
 ## Installation and Running
 Make sure the required [dependencies](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Dependencies) are met and follow the instructions available for both [NVidia](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Install-and-Run-on-NVidia-GPUs) (recommended) and [AMD](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Install-and-Run-on-AMD-GPUs) GPUs.
@@ -100,12 +99,17 @@ Alternatively, use online services (like Google Colab):
 
 - [List of Online Services](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Online-Services)
 
+### Installation on Windows 10/11 with NVidia-GPUs using release package
+1. Download `sd.webui.zip` from [v1.0.0-pre](https://github.com/AUTOMATIC1111/stable-diffusion-webui/releases/tag/v1.0.0-pre) and extract it's contents.
+2. Run `update.bat`.
+3. Run `run.bat`.
+> For more details see [Install-and-Run-on-NVidia-GPUs](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Install-and-Run-on-NVidia-GPUs)
+
 ### Automatic Installation on Windows
-1. Install [Python 3.10.6](https://www.python.org/downloads/windows/), checking "Add Python to PATH"
+1. Install [Python 3.10.6](https://www.python.org/downloads/release/python-3106/) (Newer version of Python does not support torch), checking "Add Python to PATH".
 2. Install [git](https://git-scm.com/download/win).
 3. Download the stable-diffusion-webui repository, for example by running `git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git`.
-4. Place stable diffusion checkpoint (`model.ckpt`) in the `models/Stable-diffusion` directory (see [dependencies](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Dependencies) for where to get it).
-5. Run `webui-user.bat` from Windows Explorer as normal, non-administrator, user.
+4. Run `webui-user.bat` from Windows Explorer as normal, non-administrator, user.
 
 ### Automatic Installation on Linux
 1. Install the dependencies:
@@ -117,11 +121,12 @@ sudo dnf install wget git python3
 # Arch-based:
 sudo pacman -S wget git python3
 ```
-2. To install in `/home/$(whoami)/stable-diffusion-webui/`, run:
+2. Navigate to the directory you would like the webui to be installed and execute the following command:
 ```bash
 bash <(wget -qO- https://raw.githubusercontent.com/AUTOMATIC1111/stable-diffusion-webui/master/webui.sh)
 ```
-
+3. Run `webui.sh`.
+4. Check `webui-user.sh` for options.
 ### Installation on Apple Silicon
 
 Find the instructions [here](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Installation-on-Apple-Silicon).
@@ -155,6 +160,10 @@ Licenses for borrowed code can be found in `Settings -> Licenses` screen, and al
 - Idea for Composable Diffusion - https://github.com/energy-based-model/Compositional-Visual-Generation-with-Composable-Diffusion-Models-PyTorch
 - xformers - https://github.com/facebookresearch/xformers
 - DeepDanbooru - interrogator for anime diffusers https://github.com/KichangKim/DeepDanbooru
+- Sampling in float32 precision from a float16 UNet - marunine for the idea, Birch-san for the example Diffusers implementation (https://github.com/Birch-san/diffusers-play/tree/92feee6)
+- Instruct pix2pix - Tim Brooks (star), Aleksander Holynski (star), Alexei A. Efros (no star) - https://github.com/timothybrooks/instruct-pix2pix
 - Security advice - RyotaK
+- UniPC sampler - Wenliang Zhao - https://github.com/wl-zhao/UniPC
+- TAESD - Ollin Boer Bohan - https://github.com/madebyollin/taesd
 - Initial Gradio script - posted on 4chan by an Anonymous user. Thank you Anonymous user.
 - (You)

configs/instruct-pix2pix.yaml

@@ -0,0 +1,98 @@
+# File modified by authors of InstructPix2Pix from original (https://github.com/CompVis/stable-diffusion).
+# See more details in LICENSE.
+
+model:
+  base_learning_rate: 1.0e-04
+  target: modules.models.diffusion.ddpm_edit.LatentDiffusion
+  params:
+    linear_start: 0.00085
+    linear_end: 0.0120
+    num_timesteps_cond: 1
+    log_every_t: 200
+    timesteps: 1000
+    first_stage_key: edited
+    cond_stage_key: edit
+    # image_size: 64
+    # image_size: 32
+    image_size: 16
+    channels: 4
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: hybrid
+    monitor: val/loss_simple_ema
+    scale_factor: 0.18215
+    use_ema: false
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 0 ]
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
+
+    unet_config:
+      target: ldm.modules.diffusionmodules.openaimodel.UNetModel
+      params:
+        image_size: 32 # unused
+        in_channels: 8
+        out_channels: 4
+        model_channels: 320
+        attention_resolutions: [ 4, 2, 1 ]
+        num_res_blocks: 2
+        channel_mult: [ 1, 2, 4, 4 ]
+        num_heads: 8
+        use_spatial_transformer: True
+        transformer_depth: 1
+        context_dim: 768
+        use_checkpoint: True
+        legacy: False
+
+    first_stage_config:
+      target: ldm.models.autoencoder.AutoencoderKL
+      params:
+        embed_dim: 4
+        monitor: val/rec_loss
+        ddconfig:
+          double_z: true
+          z_channels: 4
+          resolution: 256
+          in_channels: 3
+          out_ch: 3
+          ch: 128
+          ch_mult:
+          - 1
+          - 2
+          - 4
+          - 4
+          num_res_blocks: 2
+          attn_resolutions: []
+          dropout: 0.0
+        lossconfig:
+          target: torch.nn.Identity
+
+    cond_stage_config:
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder
+
+data:
+  target: main.DataModuleFromConfig
+  params:
+    batch_size: 128
+    num_workers: 1
+    wrap: false
+    validation:
+      target: edit_dataset.EditDataset
+      params:
+        path: data/clip-filtered-dataset
+        cache_dir:  data/
+        cache_name: data_10k
+        split: val
+        min_text_sim: 0.2
+        min_image_sim: 0.75
+        min_direction_sim: 0.2
+        max_samples_per_prompt: 1
+        min_resize_res: 512
+        max_resize_res: 512
+        crop_res: 512
+        output_as_edit: False
+        real_input: True

configs/v1-inpainting-inference.yaml

@@ -1,8 +1,7 @@
 model:
-  base_learning_rate: 1.0e-4
-  target: ldm.models.diffusion.ddpm.LatentDiffusion
+  base_learning_rate: 7.5e-05
+  target: ldm.models.diffusion.ddpm.LatentInpaintDiffusion
   params:
-    parameterization: "v"
     linear_start: 0.00085
     linear_end: 0.0120
     num_timesteps_cond: 1
@@ -12,29 +11,36 @@ model:
     cond_stage_key: "txt"
     image_size: 64
     channels: 4
-    cond_stage_trainable: false
-    conditioning_key: crossattn
+    cond_stage_trainable: false   # Note: different from the one we trained before
+    conditioning_key: hybrid   # important
     monitor: val/loss_simple_ema
     scale_factor: 0.18215
-    use_ema: False # we set this to false because this is an inference only config
+    finetune_keys: null
+
+    scheduler_config: # 10000 warmup steps
+      target: ldm.lr_scheduler.LambdaLinearScheduler
+      params:
+        warm_up_steps: [ 2500 ] # NOTE for resuming. use 10000 if starting from scratch
+        cycle_lengths: [ 10000000000000 ] # incredibly large number to prevent corner cases
+        f_start: [ 1.e-6 ]
+        f_max: [ 1. ]
+        f_min: [ 1. ]
 
     unet_config:
       target: ldm.modules.diffusionmodules.openaimodel.UNetModel
       params:
-        use_checkpoint: True
-        use_fp16: True
         image_size: 32 # unused
-        in_channels: 4
+        in_channels: 9  # 4 data + 4 downscaled image + 1 mask
         out_channels: 4
         model_channels: 320
         attention_resolutions: [ 4, 2, 1 ]
         num_res_blocks: 2
         channel_mult: [ 1, 2, 4, 4 ]
-        num_head_channels: 64 # need to fix for flash-attn
+        num_heads: 8
         use_spatial_transformer: True
-        use_linear_in_transformer: True
         transformer_depth: 1
-        context_dim: 1024
+        context_dim: 768
+        use_checkpoint: True
         legacy: False
 
     first_stage_config:
@@ -43,7 +49,6 @@ model:
         embed_dim: 4
         monitor: val/rec_loss
         ddconfig:
-          #attn_type: "vanilla-xformers"
           double_z: true
           z_channels: 4
           resolution: 256
@@ -62,7 +67,4 @@ model:
           target: torch.nn.Identity
 
     cond_stage_config:
-      target: ldm.modules.encoders.modules.FrozenOpenCLIPEmbedder
-      params:
-        freeze: True
-        layer: "penultimate"
+      target: ldm.modules.encoders.modules.FrozenCLIPEmbedder

environment-wsl2.yaml

@@ -4,8 +4,8 @@ channels:
   - defaults
 dependencies:
   - python=3.10
-  - pip=22.2.2
-  - cudatoolkit=11.3
-  - pytorch=1.12.1
-  - torchvision=0.13.1
-  - numpy=1.23.1
+  - pip=23.0
+  - cudatoolkit=11.8
+  - pytorch=2.0
+  - torchvision=0.15
+  - numpy=1.23

extensions-builtin/LDSR/ldsr_model_arch.py

@@ -88,7 +88,7 @@ class LDSR:
 
         x_t = None
         logs = None
-        for n in range(n_runs):
+        for _ in range(n_runs):
             if custom_shape is not None:
                 x_t = torch.randn(1, custom_shape[1], custom_shape[2], custom_shape[3]).to(model.device)
                 x_t = repeat(x_t, '1 c h w -> b c h w', b=custom_shape[0])
@@ -110,7 +110,6 @@ class LDSR:
         diffusion_steps = int(steps)
         eta = 1.0
 
-        down_sample_method = 'Lanczos'
 
         gc.collect()
         if torch.cuda.is_available:
@@ -131,11 +130,11 @@ class LDSR:
             im_og = im_og.resize((width_downsampled_pre, height_downsampled_pre), Image.LANCZOS)
         else:
             print(f"Down sample rate is 1 from {target_scale} / 4 (Not downsampling)")
-        
+
         # pad width and height to multiples of 64, pads with the edge values of image to avoid artifacts
         pad_w, pad_h = np.max(((2, 2), np.ceil(np.array(im_og.size) / 64).astype(int)), axis=0) * 64 - im_og.size
         im_padded = Image.fromarray(np.pad(np.array(im_og), ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))
-        
+
         logs = self.run(model["model"], im_padded, diffusion_steps, eta)
 
         sample = logs["sample"]
@@ -158,7 +157,7 @@ class LDSR:
 
 
 def get_cond(selected_path):
-    example = dict()
+    example = {}
     up_f = 4
     c = selected_path.convert('RGB')
     c = torch.unsqueeze(torchvision.transforms.ToTensor()(c), 0)
@@ -196,7 +195,7 @@ def convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_s
 @torch.no_grad()
 def make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize_x0=False, custom_shape=None, temperature=1., noise_dropout=0., corrector=None,
                               corrector_kwargs=None, x_T=None, ddim_use_x0_pred=False):
-    log = dict()
+    log = {}
 
     z, c, x, xrec, xc = model.get_input(batch, model.first_stage_key,
                                         return_first_stage_outputs=True,
@@ -244,7 +243,7 @@ def make_convolutional_sample(batch, model, custom_steps=None, eta=1.0, quantize
         x_sample_noquant = model.decode_first_stage(sample, force_not_quantize=True)
         log["sample_noquant"] = x_sample_noquant
         log["sample_diff"] = torch.abs(x_sample_noquant - x_sample)
-    except:
+    except Exception:
         pass
 
     log["sample"] = x_sample

extensions-builtin/LDSR/scripts/ldsr_model.py

@@ -1,13 +1,12 @@
 import os
-import sys
-import traceback
 
 from basicsr.utils.download_util import load_file_from_url
 
 from modules.upscaler import Upscaler, UpscalerData
 from ldsr_model_arch import LDSR
-from modules import shared, script_callbacks
-import sd_hijack_autoencoder, sd_hijack_ddpm_v1
+from modules import shared, script_callbacks, errors
+import sd_hijack_autoencoder  # noqa: F401
+import sd_hijack_ddpm_v1  # noqa: F401
 
 
 class UpscalerLDSR(Upscaler):
@@ -25,29 +24,33 @@ class UpscalerLDSR(Upscaler):
         yaml_path = os.path.join(self.model_path, "project.yaml")
         old_model_path = os.path.join(self.model_path, "model.pth")
         new_model_path = os.path.join(self.model_path, "model.ckpt")
-        safetensors_model_path = os.path.join(self.model_path, "model.safetensors")
+
+        local_model_paths = self.find_models(ext_filter=[".ckpt", ".safetensors"])
+        local_ckpt_path = next(iter([local_model for local_model in local_model_paths if local_model.endswith("model.ckpt")]), None)
+        local_safetensors_path = next(iter([local_model for local_model in local_model_paths if local_model.endswith("model.safetensors")]), None)
+        local_yaml_path = next(iter([local_model for local_model in local_model_paths if local_model.endswith("project.yaml")]), None)
+
         if os.path.exists(yaml_path):
             statinfo = os.stat(yaml_path)
             if statinfo.st_size >= 10485760:
                 print("Removing invalid LDSR YAML file.")
                 os.remove(yaml_path)
+
         if os.path.exists(old_model_path):
             print("Renaming model from model.pth to model.ckpt")
             os.rename(old_model_path, new_model_path)
-        if os.path.exists(safetensors_model_path):
-            model = safetensors_model_path
+
+        if local_safetensors_path is not None and os.path.exists(local_safetensors_path):
+            model = local_safetensors_path
         else:
-            model = load_file_from_url(url=self.model_url, model_dir=self.model_path,
-                                       file_name="model.ckpt", progress=True)
-        yaml = load_file_from_url(url=self.yaml_url, model_dir=self.model_path,
-                                  file_name="project.yaml", progress=True)
+            model = local_ckpt_path if local_ckpt_path is not None else load_file_from_url(url=self.model_url, model_dir=self.model_download_path, file_name="model.ckpt", progress=True)
+
+        yaml = local_yaml_path if local_yaml_path is not None else load_file_from_url(url=self.yaml_url, model_dir=self.model_download_path, file_name="project.yaml", progress=True)
 
         try:
             return LDSR(model, yaml)
-
         except Exception:
-            print("Error importing LDSR:", file=sys.stderr)
-            print(traceback.format_exc(), file=sys.stderr)
+            errors.report("Error importing LDSR", exc_info=True)
         return None
 
     def do_upscale(self, img, path):

extensions-builtin/LDSR/sd_hijack_autoencoder.py

@@ -1,16 +1,21 @@
 # The content of this file comes from the ldm/models/autoencoder.py file of the compvis/stable-diffusion repo
 # The VQModel & VQModelInterface were subsequently removed from ldm/models/autoencoder.py when we moved to the stability-ai/stablediffusion repo
 # As the LDSR upscaler relies on VQModel & VQModelInterface, the hijack aims to put them back into the ldm.models.autoencoder
-
+import numpy as np
 import torch
 import pytorch_lightning as pl
 import torch.nn.functional as F
 from contextlib import contextmanager
-from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer
+
+from torch.optim.lr_scheduler import LambdaLR
+
+from ldm.modules.ema import LitEma
+from vqvae_quantize import VectorQuantizer2 as VectorQuantizer
 from ldm.modules.diffusionmodules.model import Encoder, Decoder
 from ldm.util import instantiate_from_config
 
 import ldm.models.autoencoder
+from packaging import version
 
 class VQModel(pl.LightningModule):
     def __init__(self,
@@ -19,7 +24,7 @@ class VQModel(pl.LightningModule):
                  n_embed,
                  embed_dim,
                  ckpt_path=None,
-                 ignore_keys=[],
+                 ignore_keys=None,
                  image_key="image",
                  colorize_nlabels=None,
                  monitor=None,
@@ -57,7 +62,7 @@ class VQModel(pl.LightningModule):
             print(f"Keeping EMAs of {len(list(self.model_ema.buffers()))}.")
 
         if ckpt_path is not None:
-            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys)
+            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys or [])
         self.scheduler_config = scheduler_config
         self.lr_g_factor = lr_g_factor
 
@@ -76,18 +81,19 @@ class VQModel(pl.LightningModule):
                 if context is not None:
                     print(f"{context}: Restored training weights")
 
-    def init_from_ckpt(self, path, ignore_keys=list()):
+    def init_from_ckpt(self, path, ignore_keys=None):
         sd = torch.load(path, map_location="cpu")["state_dict"]
         keys = list(sd.keys())
         for k in keys:
-            for ik in ignore_keys:
+            for ik in ignore_keys or []:
                 if k.startswith(ik):
                     print("Deleting key {} from state_dict.".format(k))
                     del sd[k]
         missing, unexpected = self.load_state_dict(sd, strict=False)
         print(f"Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys")
-        if len(missing) > 0:
+        if missing:
             print(f"Missing Keys: {missing}")
+        if unexpected:
             print(f"Unexpected Keys: {unexpected}")
 
     def on_train_batch_end(self, *args, **kwargs):
@@ -165,7 +171,7 @@ class VQModel(pl.LightningModule):
     def validation_step(self, batch, batch_idx):
         log_dict = self._validation_step(batch, batch_idx)
         with self.ema_scope():
-            log_dict_ema = self._validation_step(batch, batch_idx, suffix="_ema")
+            self._validation_step(batch, batch_idx, suffix="_ema")
         return log_dict
 
     def _validation_step(self, batch, batch_idx, suffix=""):
@@ -232,7 +238,7 @@ class VQModel(pl.LightningModule):
         return self.decoder.conv_out.weight
 
     def log_images(self, batch, only_inputs=False, plot_ema=False, **kwargs):
-        log = dict()
+        log = {}
         x = self.get_input(batch, self.image_key)
         x = x.to(self.device)
         if only_inputs:
@@ -249,7 +255,8 @@ class VQModel(pl.LightningModule):
         if plot_ema:
             with self.ema_scope():
                 xrec_ema, _ = self(x)
-                if x.shape[1] > 3: xrec_ema = self.to_rgb(xrec_ema)
+                if x.shape[1] > 3:
+                    xrec_ema = self.to_rgb(xrec_ema)
                 log["reconstructions_ema"] = xrec_ema
         return log
 
@@ -264,7 +271,7 @@ class VQModel(pl.LightningModule):
 
 class VQModelInterface(VQModel):
     def __init__(self, embed_dim, *args, **kwargs):
-        super().__init__(embed_dim=embed_dim, *args, **kwargs)
+        super().__init__(*args, embed_dim=embed_dim, **kwargs)
         self.embed_dim = embed_dim
 
     def encode(self, x):
@@ -282,5 +289,5 @@ class VQModelInterface(VQModel):
         dec = self.decoder(quant)
         return dec
 
-setattr(ldm.models.autoencoder, "VQModel", VQModel)
-setattr(ldm.models.autoencoder, "VQModelInterface", VQModelInterface)
+ldm.models.autoencoder.VQModel = VQModel
+ldm.models.autoencoder.VQModelInterface = VQModelInterface

extensions-builtin/LDSR/sd_hijack_ddpm_v1.py

@@ -48,7 +48,7 @@ class DDPMV1(pl.LightningModule):
                  beta_schedule="linear",
                  loss_type="l2",
                  ckpt_path=None,
-                 ignore_keys=[],
+                 ignore_keys=None,
                  load_only_unet=False,
                  monitor="val/loss",
                  use_ema=True,
@@ -100,7 +100,7 @@ class DDPMV1(pl.LightningModule):
         if monitor is not None:
             self.monitor = monitor
         if ckpt_path is not None:
-            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, only_model=load_only_unet)
+            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys or [], only_model=load_only_unet)
 
         self.register_schedule(given_betas=given_betas, beta_schedule=beta_schedule, timesteps=timesteps,
                                linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s)
@@ -182,22 +182,22 @@ class DDPMV1(pl.LightningModule):
                 if context is not None:
                     print(f"{context}: Restored training weights")
 
-    def init_from_ckpt(self, path, ignore_keys=list(), only_model=False):
+    def init_from_ckpt(self, path, ignore_keys=None, only_model=False):
         sd = torch.load(path, map_location="cpu")
         if "state_dict" in list(sd.keys()):
             sd = sd["state_dict"]
         keys = list(sd.keys())
         for k in keys:
-            for ik in ignore_keys:
+            for ik in ignore_keys or []:
                 if k.startswith(ik):
                     print("Deleting key {} from state_dict.".format(k))
                     del sd[k]
         missing, unexpected = self.load_state_dict(sd, strict=False) if not only_model else self.model.load_state_dict(
             sd, strict=False)
         print(f"Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys")
-        if len(missing) > 0:
+        if missing:
             print(f"Missing Keys: {missing}")
-        if len(unexpected) > 0:
+        if unexpected:
             print(f"Unexpected Keys: {unexpected}")
 
     def q_mean_variance(self, x_start, t):
@@ -375,7 +375,7 @@ class DDPMV1(pl.LightningModule):
 
     @torch.no_grad()
     def log_images(self, batch, N=8, n_row=2, sample=True, return_keys=None, **kwargs):
-        log = dict()
+        log = {}
         x = self.get_input(batch, self.first_stage_key)
         N = min(x.shape[0], N)
         n_row = min(x.shape[0], n_row)
@@ -383,7 +383,7 @@ class DDPMV1(pl.LightningModule):
         log["inputs"] = x
 
         # get diffusion row
-        diffusion_row = list()
+        diffusion_row = []
         x_start = x[:n_row]
 
         for t in range(self.num_timesteps):
@@ -444,13 +444,13 @@ class LatentDiffusionV1(DDPMV1):
             conditioning_key = None
         ckpt_path = kwargs.pop("ckpt_path", None)
         ignore_keys = kwargs.pop("ignore_keys", [])
-        super().__init__(conditioning_key=conditioning_key, *args, **kwargs)
+        super().__init__(*args, conditioning_key=conditioning_key, **kwargs)
         self.concat_mode = concat_mode
         self.cond_stage_trainable = cond_stage_trainable
         self.cond_stage_key = cond_stage_key
         try:
             self.num_downs = len(first_stage_config.params.ddconfig.ch_mult) - 1
-        except:
+        except Exception:
             self.num_downs = 0
         if not scale_by_std:
             self.scale_factor = scale_factor
@@ -460,7 +460,7 @@ class LatentDiffusionV1(DDPMV1):
         self.instantiate_cond_stage(cond_stage_config)
         self.cond_stage_forward = cond_stage_forward
         self.clip_denoised = False
-        self.bbox_tokenizer = None  
+        self.bbox_tokenizer = None
 
         self.restarted_from_ckpt = False
         if ckpt_path is not None:
@@ -792,7 +792,7 @@ class LatentDiffusionV1(DDPMV1):
                 z = z.view((z.shape[0], -1, ks[0], ks[1], z.shape[-1]))  # (bn, nc, ks[0], ks[1], L )
 
                 # 2. apply model loop over last dim
-                if isinstance(self.first_stage_model, VQModelInterface):  
+                if isinstance(self.first_stage_model, VQModelInterface):
                     output_list = [self.first_stage_model.decode(z[:, :, :, :, i],
                                                                  force_not_quantize=predict_cids or force_not_quantize)
                                    for i in range(z.shape[-1])]
@@ -877,16 +877,6 @@ class LatentDiffusionV1(DDPMV1):
                 c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float()))
         return self.p_losses(x, c, t, *args, **kwargs)
 
-    def _rescale_annotations(self, bboxes, crop_coordinates):  # TODO: move to dataset
-        def rescale_bbox(bbox):
-            x0 = clamp((bbox[0] - crop_coordinates[0]) / crop_coordinates[2])
-            y0 = clamp((bbox[1] - crop_coordinates[1]) / crop_coordinates[3])
-            w = min(bbox[2] / crop_coordinates[2], 1 - x0)
-            h = min(bbox[3] / crop_coordinates[3], 1 - y0)
-            return x0, y0, w, h
-
-        return [rescale_bbox(b) for b in bboxes]
-
     def apply_model(self, x_noisy, t, cond, return_ids=False):
 
         if isinstance(cond, dict):
@@ -900,7 +890,7 @@ class LatentDiffusionV1(DDPMV1):
 
         if hasattr(self, "split_input_params"):
             assert len(cond) == 1  # todo can only deal with one conditioning atm
-            assert not return_ids  
+            assert not return_ids
             ks = self.split_input_params["ks"]  # eg. (128, 128)
             stride = self.split_input_params["stride"]  # eg. (64, 64)
 
@@ -1126,7 +1116,7 @@ class LatentDiffusionV1(DDPMV1):
         if cond is not None:
             if isinstance(cond, dict):
                 cond = {key: cond[key][:batch_size] if not isinstance(cond[key], list) else
-                list(map(lambda x: x[:batch_size], cond[key])) for key in cond}
+                [x[:batch_size] for x in cond[key]] for key in cond}
             else:
                 cond = [c[:batch_size] for c in cond] if isinstance(cond, list) else cond[:batch_size]
 
@@ -1157,8 +1147,10 @@ class LatentDiffusionV1(DDPMV1):
 
             if i % log_every_t == 0 or i == timesteps - 1:
                 intermediates.append(x0_partial)
-            if callback: callback(i)
-            if img_callback: img_callback(img, i)
+            if callback:
+                callback(i)
+            if img_callback:
+                img_callback(img, i)
         return img, intermediates
 
     @torch.no_grad()
@@ -1205,8 +1197,10 @@ class LatentDiffusionV1(DDPMV1):
 
             if i % log_every_t == 0 or i == timesteps - 1:
                 intermediates.append(img)
-            if callback: callback(i)
-            if img_callback: img_callback(img, i)
+            if callback:
+                callback(i)
+            if img_callback:
+                img_callback(img, i)
 
         if return_intermediates:
             return img, intermediates
@@ -1221,7 +1215,7 @@ class LatentDiffusionV1(DDPMV1):
         if cond is not None:
             if isinstance(cond, dict):
                 cond = {key: cond[key][:batch_size] if not isinstance(cond[key], list) else
-                list(map(lambda x: x[:batch_size], cond[key])) for key in cond}
+                [x[:batch_size] for x in cond[key]] for key in cond}
             else:
                 cond = [c[:batch_size] for c in cond] if isinstance(cond, list) else cond[:batch_size]
         return self.p_sample_loop(cond,
@@ -1253,7 +1247,7 @@ class LatentDiffusionV1(DDPMV1):
 
         use_ddim = ddim_steps is not None
 
-        log = dict()
+        log = {}
         z, c, x, xrec, xc = self.get_input(batch, self.first_stage_key,
                                            return_first_stage_outputs=True,
                                            force_c_encode=True,
@@ -1280,7 +1274,7 @@ class LatentDiffusionV1(DDPMV1):
 
         if plot_diffusion_rows:
             # get diffusion row
-            diffusion_row = list()
+            diffusion_row = []
             z_start = z[:n_row]
             for t in range(self.num_timesteps):
                 if t % self.log_every_t == 0 or t == self.num_timesteps - 1:
@@ -1322,7 +1316,7 @@ class LatentDiffusionV1(DDPMV1):
 
             if inpaint:
                 # make a simple center square
-                b, h, w = z.shape[0], z.shape[2], z.shape[3]
+                h, w = z.shape[2], z.shape[3]
                 mask = torch.ones(N, h, w).to(self.device)
                 # zeros will be filled in
                 mask[:, h // 4:3 * h // 4, w // 4:3 * w // 4] = 0.
@@ -1424,10 +1418,10 @@ class Layout2ImgDiffusionV1(LatentDiffusionV1):
     # TODO: move all layout-specific hacks to this class
     def __init__(self, cond_stage_key, *args, **kwargs):
         assert cond_stage_key == 'coordinates_bbox', 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"'
-        super().__init__(cond_stage_key=cond_stage_key, *args, **kwargs)
+        super().__init__(*args, cond_stage_key=cond_stage_key, **kwargs)
 
     def log_images(self, batch, N=8, *args, **kwargs):
-        logs = super().log_images(batch=batch, N=N, *args, **kwargs)
+        logs = super().log_images(*args, batch=batch, N=N, **kwargs)
 
         key = 'train' if self.training else 'validation'
         dset = self.trainer.datamodule.datasets[key]
@@ -1443,7 +1437,7 @@ class Layout2ImgDiffusionV1(LatentDiffusionV1):
         logs['bbox_image'] = cond_img
         return logs
 
-setattr(ldm.models.diffusion.ddpm, "DDPMV1", DDPMV1)
-setattr(ldm.models.diffusion.ddpm, "LatentDiffusionV1", LatentDiffusionV1)
-setattr(ldm.models.diffusion.ddpm, "DiffusionWrapperV1", DiffusionWrapperV1)
-setattr(ldm.models.diffusion.ddpm, "Layout2ImgDiffusionV1", Layout2ImgDiffusionV1)
+ldm.models.diffusion.ddpm.DDPMV1 = DDPMV1
+ldm.models.diffusion.ddpm.LatentDiffusionV1 = LatentDiffusionV1
+ldm.models.diffusion.ddpm.DiffusionWrapperV1 = DiffusionWrapperV1
+ldm.models.diffusion.ddpm.Layout2ImgDiffusionV1 = Layout2ImgDiffusionV1

extensions-builtin/LDSR/vqvae_quantize.py

@@ -0,0 +1,147 @@
+# Vendored from https://raw.githubusercontent.com/CompVis/taming-transformers/24268930bf1dce879235a7fddd0b2355b84d7ea6/taming/modules/vqvae/quantize.py,
+# where the license is as follows:
+#
+# Copyright (c) 2020 Patrick Esser and Robin Rombach and Björn Ommer
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
+# OR OTHER DEALINGS IN THE SOFTWARE./
+
+import torch
+import torch.nn as nn
+import numpy as np
+from einops import rearrange
+
+
+class VectorQuantizer2(nn.Module):
+    """
+    Improved version over VectorQuantizer, can be used as a drop-in replacement. Mostly
+    avoids costly matrix multiplications and allows for post-hoc remapping of indices.
+    """
+
+    # NOTE: due to a bug the beta term was applied to the wrong term. for
+    # backwards compatibility we use the buggy version by default, but you can
+    # specify legacy=False to fix it.
+    def __init__(self, n_e, e_dim, beta, remap=None, unknown_index="random",
+                 sane_index_shape=False, legacy=True):
+        super().__init__()
+        self.n_e = n_e
+        self.e_dim = e_dim
+        self.beta = beta
+        self.legacy = legacy
+
+        self.embedding = nn.Embedding(self.n_e, self.e_dim)
+        self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e)
+
+        self.remap = remap
+        if self.remap is not None:
+            self.register_buffer("used", torch.tensor(np.load(self.remap)))
+            self.re_embed = self.used.shape[0]
+            self.unknown_index = unknown_index  # "random" or "extra" or integer
+            if self.unknown_index == "extra":
+                self.unknown_index = self.re_embed
+                self.re_embed = self.re_embed + 1
+            print(f"Remapping {self.n_e} indices to {self.re_embed} indices. "
+                  f"Using {self.unknown_index} for unknown indices.")
+        else:
+            self.re_embed = n_e
+
+        self.sane_index_shape = sane_index_shape
+
+    def remap_to_used(self, inds):
+        ishape = inds.shape
+        assert len(ishape) > 1
+        inds = inds.reshape(ishape[0], -1)
+        used = self.used.to(inds)
+        match = (inds[:, :, None] == used[None, None, ...]).long()
+        new = match.argmax(-1)
+        unknown = match.sum(2) < 1
+        if self.unknown_index == "random":
+            new[unknown] = torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device)
+        else:
+            new[unknown] = self.unknown_index
+        return new.reshape(ishape)
+
+    def unmap_to_all(self, inds):
+        ishape = inds.shape
+        assert len(ishape) > 1
+        inds = inds.reshape(ishape[0], -1)
+        used = self.used.to(inds)
+        if self.re_embed > self.used.shape[0]:  # extra token
+            inds[inds >= self.used.shape[0]] = 0  # simply set to zero
+        back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds)
+        return back.reshape(ishape)
+
+    def forward(self, z, temp=None, rescale_logits=False, return_logits=False):
+        assert temp is None or temp == 1.0, "Only for interface compatible with Gumbel"
+        assert rescale_logits is False, "Only for interface compatible with Gumbel"
+        assert return_logits is False, "Only for interface compatible with Gumbel"
+        # reshape z -> (batch, height, width, channel) and flatten
+        z = rearrange(z, 'b c h w -> b h w c').contiguous()
+        z_flattened = z.view(-1, self.e_dim)
+        # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
+
+        d = torch.sum(z_flattened ** 2, dim=1, keepdim=True) + \
+            torch.sum(self.embedding.weight ** 2, dim=1) - 2 * \
+            torch.einsum('bd,dn->bn', z_flattened, rearrange(self.embedding.weight, 'n d -> d n'))
+
+        min_encoding_indices = torch.argmin(d, dim=1)
+        z_q = self.embedding(min_encoding_indices).view(z.shape)
+        perplexity = None
+        min_encodings = None
+
+        # compute loss for embedding
+        if not self.legacy:
+            loss = self.beta * torch.mean((z_q.detach() - z) ** 2) + \
+                   torch.mean((z_q - z.detach()) ** 2)
+        else:
+            loss = torch.mean((z_q.detach() - z) ** 2) + self.beta * \
+                   torch.mean((z_q - z.detach()) ** 2)
+
+        # preserve gradients
+        z_q = z + (z_q - z).detach()
+
+        # reshape back to match original input shape
+        z_q = rearrange(z_q, 'b h w c -> b c h w').contiguous()
+
+        if self.remap is not None:
+            min_encoding_indices = min_encoding_indices.reshape(z.shape[0], -1)  # add batch axis
+            min_encoding_indices = self.remap_to_used(min_encoding_indices)
+            min_encoding_indices = min_encoding_indices.reshape(-1, 1)  # flatten
+
+        if self.sane_index_shape:
+            min_encoding_indices = min_encoding_indices.reshape(
+                z_q.shape[0], z_q.shape[2], z_q.shape[3])
+
+        return z_q, loss, (perplexity, min_encodings, min_encoding_indices)
+
+    def get_codebook_entry(self, indices, shape):
+        # shape specifying (batch, height, width, channel)
+        if self.remap is not None:
+            indices = indices.reshape(shape[0], -1)  # add batch axis
+            indices = self.unmap_to_all(indices)
+            indices = indices.reshape(-1)  # flatten again
+
+        # get quantized latent vectors
+        z_q = self.embedding(indices)
+
+        if shape is not None:
+            z_q = z_q.view(shape)
+            # reshape back to match original input shape
+            z_q = z_q.permute(0, 3, 1, 2).contiguous()
+
+        return z_q

extensions-builtin/Lora/extra_networks_lora.py

@@ -1,20 +1,45 @@
-from modules import extra_networks
+from modules import extra_networks, shared
 import lora
 
+
 class ExtraNetworkLora(extra_networks.ExtraNetwork):
     def __init__(self):
         super().__init__('lora')
 
     def activate(self, p, params_list):
+        additional = shared.opts.sd_lora
+
+        if additional != "None" and additional in lora.available_loras and not any(x for x in params_list if x.items[0] == additional):
+            p.all_prompts = [x + f"<lora:{additional}:{shared.opts.extra_networks_default_multiplier}>" for x in p.all_prompts]
+            params_list.append(extra_networks.ExtraNetworkParams(items=[additional, shared.opts.extra_networks_default_multiplier]))
+
         names = []
         multipliers = []
         for params in params_list:
-            assert len(params.items) > 0
+            assert params.items
 
             names.append(params.items[0])
             multipliers.append(float(params.items[1]) if len(params.items) > 1 else 1.0)
 
         lora.load_loras(names, multipliers)
 
+        if shared.opts.lora_add_hashes_to_infotext:
+            lora_hashes = []
+            for item in lora.loaded_loras:
+                shorthash = item.lora_on_disk.shorthash
+                if not shorthash:
+                    continue
+
+                alias = item.mentioned_name
+                if not alias:
+                    continue
+
+                alias = alias.replace(":", "").replace(",", "")
+
+                lora_hashes.append(f"{alias}: {shorthash}")
+
+            if lora_hashes:
+                p.extra_generation_params["Lora hashes"] = ", ".join(lora_hashes)
+
     def deactivate(self, p):
         pass

extensions-builtin/Lora/lora.py

@@ -1,19 +1,34 @@
-import glob
 import os
 import re
 import torch
+from typing import Union
 
-from modules import shared, devices, sd_models
+from modules import shared, devices, sd_models, errors, scripts, sd_hijack, hashes
+
+metadata_tags_order = {"ss_sd_model_name": 1, "ss_resolution": 2, "ss_clip_skip": 3, "ss_num_train_images": 10, "ss_tag_frequency": 20}
 
 re_digits = re.compile(r"\d+")
-re_unet_down_blocks = re.compile(r"lora_unet_down_blocks_(\d+)_attentions_(\d+)_(.+)")
-re_unet_mid_blocks = re.compile(r"lora_unet_mid_block_attentions_(\d+)_(.+)")
-re_unet_up_blocks = re.compile(r"lora_unet_up_blocks_(\d+)_attentions_(\d+)_(.+)")
-re_text_block = re.compile(r"lora_te_text_model_encoder_layers_(\d+)_(.+)")
+re_x_proj = re.compile(r"(.*)_([qkv]_proj)$")
+re_compiled = {}
+
+suffix_conversion = {
+    "attentions": {},
+    "resnets": {
+        "conv1": "in_layers_2",
+        "conv2": "out_layers_3",
+        "time_emb_proj": "emb_layers_1",
+        "conv_shortcut": "skip_connection",
+    }
+}
 
 
-def convert_diffusers_name_to_compvis(key):
-    def match(match_list, regex):
+def convert_diffusers_name_to_compvis(key, is_sd2):
+    def match(match_list, regex_text):
+        regex = re_compiled.get(regex_text)
+        if regex is None:
+            regex = re.compile(regex_text)
+            re_compiled[regex_text] = regex
+
         r = re.match(regex, key)
         if not r:
             return False
@@ -24,16 +39,33 @@ def convert_diffusers_name_to_compvis(key):
 
     m = []
 
-    if match(m, re_unet_down_blocks):
-        return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[1]}_1_{m[2]}"
+    if match(m, r"lora_unet_down_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
+        return f"diffusion_model_input_blocks_{1 + m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
 
-    if match(m, re_unet_mid_blocks):
-        return f"diffusion_model_middle_block_1_{m[1]}"
+    if match(m, r"lora_unet_mid_block_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[0], {}).get(m[2], m[2])
+        return f"diffusion_model_middle_block_{1 if m[0] == 'attentions' else m[1] * 2}_{suffix}"
 
-    if match(m, re_unet_up_blocks):
-        return f"diffusion_model_output_blocks_{m[0] * 3 + m[1]}_1_{m[2]}"
+    if match(m, r"lora_unet_up_blocks_(\d+)_(attentions|resnets)_(\d+)_(.+)"):
+        suffix = suffix_conversion.get(m[1], {}).get(m[3], m[3])
+        return f"diffusion_model_output_blocks_{m[0] * 3 + m[2]}_{1 if m[1] == 'attentions' else 0}_{suffix}"
+
+    if match(m, r"lora_unet_down_blocks_(\d+)_downsamplers_0_conv"):
+        return f"diffusion_model_input_blocks_{3 + m[0] * 3}_0_op"
+
+    if match(m, r"lora_unet_up_blocks_(\d+)_upsamplers_0_conv"):
+        return f"diffusion_model_output_blocks_{2 + m[0] * 3}_{2 if m[0]>0 else 1}_conv"
+
+    if match(m, r"lora_te_text_model_encoder_layers_(\d+)_(.+)"):
+        if is_sd2:
+            if 'mlp_fc1' in m[1]:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc1', 'mlp_c_fc')}"
+            elif 'mlp_fc2' in m[1]:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('mlp_fc2', 'mlp_c_proj')}"
+            else:
+                return f"model_transformer_resblocks_{m[0]}_{m[1].replace('self_attn', 'attn')}"
 
-    if match(m, re_text_block):
         return f"transformer_text_model_encoder_layers_{m[0]}_{m[1]}"
 
     return key
@@ -43,15 +75,62 @@ class LoraOnDisk:
     def __init__(self, name, filename):
         self.name = name
         self.filename = filename
+        self.metadata = {}
+        self.is_safetensors = os.path.splitext(filename)[1].lower() == ".safetensors"
+
+        if self.is_safetensors:
+            try:
+                self.metadata = sd_models.read_metadata_from_safetensors(filename)
+            except Exception as e:
+                errors.display(e, f"reading lora {filename}")
+
+        if self.metadata:
+            m = {}
+            for k, v in sorted(self.metadata.items(), key=lambda x: metadata_tags_order.get(x[0], 999)):
+                m[k] = v
+
+            self.metadata = m
+
+        self.ssmd_cover_images = self.metadata.pop('ssmd_cover_images', None)  # those are cover images and they are too big to display in UI as text
+        self.alias = self.metadata.get('ss_output_name', self.name)
+
+        self.hash = None
+        self.shorthash = None
+        self.set_hash(
+            self.metadata.get('sshs_model_hash') or
+            hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or
+            ''
+        )
+
+    def set_hash(self, v):
+        self.hash = v
+        self.shorthash = self.hash[0:12]
+
+        if self.shorthash:
+            available_lora_hash_lookup[self.shorthash] = self
+
+    def read_hash(self):
+        if not self.hash:
+            self.set_hash(hashes.sha256(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '')
+
+    def get_alias(self):
+        if shared.opts.lora_preferred_name == "Filename" or self.alias.lower() in forbidden_lora_aliases:
+            return self.name
+        else:
+            return self.alias
 
 
 class LoraModule:
-    def __init__(self, name):
+    def __init__(self, name, lora_on_disk: LoraOnDisk):
         self.name = name
+        self.lora_on_disk = lora_on_disk
         self.multiplier = 1.0
         self.modules = {}
         self.mtime = None
 
+        self.mentioned_name = None
+        """the text that was used to add lora to prompt - can be either name or an alias"""
+
 
 class LoraUpDownModule:
     def __init__(self):
@@ -76,21 +155,32 @@ def assign_lora_names_to_compvis_modules(sd_model):
     sd_model.lora_layer_mapping = lora_layer_mapping
 
 
-def load_lora(name, filename):
-    lora = LoraModule(name)
-    lora.mtime = os.path.getmtime(filename)
+def load_lora(name, lora_on_disk):
+    lora = LoraModule(name, lora_on_disk)
+    lora.mtime = os.path.getmtime(lora_on_disk.filename)
 
-    sd = sd_models.read_state_dict(filename)
+    sd = sd_models.read_state_dict(lora_on_disk.filename)
 
-    keys_failed_to_match = []
+    # this should not be needed but is here as an emergency fix for an unknown error people are experiencing in 1.2.0
+    if not hasattr(shared.sd_model, 'lora_layer_mapping'):
+        assign_lora_names_to_compvis_modules(shared.sd_model)
+
+    keys_failed_to_match = {}
+    is_sd2 = 'model_transformer_resblocks' in shared.sd_model.lora_layer_mapping
 
     for key_diffusers, weight in sd.items():
-        fullkey = convert_diffusers_name_to_compvis(key_diffusers)
-        key, lora_key = fullkey.split(".", 1)
+        key_diffusers_without_lora_parts, lora_key = key_diffusers.split(".", 1)
+        key = convert_diffusers_name_to_compvis(key_diffusers_without_lora_parts, is_sd2)
 
         sd_module = shared.sd_model.lora_layer_mapping.get(key, None)
+
         if sd_module is None:
-            keys_failed_to_match.append(key_diffusers)
+            m = re_x_proj.match(key)
+            if m:
+                sd_module = shared.sd_model.lora_layer_mapping.get(m.group(1), None)
+
+        if sd_module is None:
+            keys_failed_to_match[key_diffusers] = key
             continue
 
         lora_module = lora.modules.get(key, None)
@@ -104,25 +194,33 @@ def load_lora(name, filename):
 
         if type(sd_module) == torch.nn.Linear:
             module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False)
-        elif type(sd_module) == torch.nn.Conv2d:
+        elif type(sd_module) == torch.nn.modules.linear.NonDynamicallyQuantizableLinear:
+            module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False)
+        elif type(sd_module) == torch.nn.MultiheadAttention:
+            module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False)
+        elif type(sd_module) == torch.nn.Conv2d and weight.shape[2:] == (1, 1):
             module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False)
+        elif type(sd_module) == torch.nn.Conv2d and weight.shape[2:] == (3, 3):
+            module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (3, 3), bias=False)
         else:
-            assert False, f'Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}'
+            print(f'Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}')
+            continue
+            raise AssertionError(f"Lora layer {key_diffusers} matched a layer with unsupported type: {type(sd_module).__name__}")
 
         with torch.no_grad():
             module.weight.copy_(weight)
 
-        module.to(device=devices.device, dtype=devices.dtype)
+        module.to(device=devices.cpu, dtype=devices.dtype)
 
         if lora_key == "lora_up.weight":
             lora_module.up = module
         elif lora_key == "lora_down.weight":
             lora_module.down = module
         else:
-            assert False, f'Bad Lora layer name: {key_diffusers} - must end in lora_up.weight, lora_down.weight or alpha'
+            raise AssertionError(f"Bad Lora layer name: {key_diffusers} - must end in lora_up.weight, lora_down.weight or alpha")
 
-    if len(keys_failed_to_match) > 0:
-        print(f"Failed to match keys when loading Lora {filename}: {keys_failed_to_match}")
+    if keys_failed_to_match:
+        print(f"Failed to match keys when loading Lora {lora_on_disk.filename}: {keys_failed_to_match}")
 
     return lora
 
@@ -136,69 +234,273 @@ def load_loras(names, multipliers=None):
 
     loaded_loras.clear()
 
-    loras_on_disk = [available_loras.get(name, None) for name in names]
-    if any([x is None for x in loras_on_disk]):
+    loras_on_disk = [available_lora_aliases.get(name, None) for name in names]
+    if any(x is None for x in loras_on_disk):
         list_available_loras()
 
-        loras_on_disk = [available_loras.get(name, None) for name in names]
+        loras_on_disk = [available_lora_aliases.get(name, None) for name in names]
+
+    failed_to_load_loras = []
 
     for i, name in enumerate(names):
         lora = already_loaded.get(name, None)
 
         lora_on_disk = loras_on_disk[i]
+
         if lora_on_disk is not None:
             if lora is None or os.path.getmtime(lora_on_disk.filename) > lora.mtime:
-                lora = load_lora(name, lora_on_disk.filename)
+                try:
+                    lora = load_lora(name, lora_on_disk)
+                except Exception as e:
+                    errors.display(e, f"loading Lora {lora_on_disk.filename}")
+                    continue
+
+            lora.mentioned_name = name
+
+            lora_on_disk.read_hash()
 
         if lora is None:
+            failed_to_load_loras.append(name)
             print(f"Couldn't find Lora with name {name}")
             continue
 
         lora.multiplier = multipliers[i] if multipliers else 1.0
         loaded_loras.append(lora)
 
+    if failed_to_load_loras:
+        sd_hijack.model_hijack.comments.append("Failed to find Loras: " + ", ".join(failed_to_load_loras))
+
+
+def lora_calc_updown(lora, module, target):
+    with torch.no_grad():
+        up = module.up.weight.to(target.device, dtype=target.dtype)
+        down = module.down.weight.to(target.device, dtype=target.dtype)
+
+        if up.shape[2:] == (1, 1) and down.shape[2:] == (1, 1):
+            updown = (up.squeeze(2).squeeze(2) @ down.squeeze(2).squeeze(2)).unsqueeze(2).unsqueeze(3)
+        elif up.shape[2:] == (3, 3) or down.shape[2:] == (3, 3):
+            updown = torch.nn.functional.conv2d(down.permute(1, 0, 2, 3), up).permute(1, 0, 2, 3)
+        else:
+            updown = up @ down
+
+        updown = updown * lora.multiplier * (module.alpha / module.up.weight.shape[1] if module.alpha else 1.0)
+
+        return updown
+
+
+def lora_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.MultiheadAttention]):
+    weights_backup = getattr(self, "lora_weights_backup", None)
+
+    if weights_backup is None:
+        return
+
+    if isinstance(self, torch.nn.MultiheadAttention):
+        self.in_proj_weight.copy_(weights_backup[0])
+        self.out_proj.weight.copy_(weights_backup[1])
+    else:
+        self.weight.copy_(weights_backup)
+
+
+def lora_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.MultiheadAttention]):
+    """
+    Applies the currently selected set of Loras to the weights of torch layer self.
+    If weights already have this particular set of loras applied, does nothing.
+    If not, restores orginal weights from backup and alters weights according to loras.
+    """
+
+    lora_layer_name = getattr(self, 'lora_layer_name', None)
+    if lora_layer_name is None:
+        return
+
+    current_names = getattr(self, "lora_current_names", ())
+    wanted_names = tuple((x.name, x.multiplier) for x in loaded_loras)
+
+    weights_backup = getattr(self, "lora_weights_backup", None)
+    if weights_backup is None:
+        if isinstance(self, torch.nn.MultiheadAttention):
+            weights_backup = (self.in_proj_weight.to(devices.cpu, copy=True), self.out_proj.weight.to(devices.cpu, copy=True))
+        else:
+            weights_backup = self.weight.to(devices.cpu, copy=True)
+
+        self.lora_weights_backup = weights_backup
+
+    if current_names != wanted_names:
+        lora_restore_weights_from_backup(self)
+
+        for lora in loaded_loras:
+            module = lora.modules.get(lora_layer_name, None)
+            if module is not None and hasattr(self, 'weight'):
+                self.weight += lora_calc_updown(lora, module, self.weight)
+                continue
+
+            module_q = lora.modules.get(lora_layer_name + "_q_proj", None)
+            module_k = lora.modules.get(lora_layer_name + "_k_proj", None)
+            module_v = lora.modules.get(lora_layer_name + "_v_proj", None)
+            module_out = lora.modules.get(lora_layer_name + "_out_proj", None)
+
+            if isinstance(self, torch.nn.MultiheadAttention) and module_q and module_k and module_v and module_out:
+                updown_q = lora_calc_updown(lora, module_q, self.in_proj_weight)
+                updown_k = lora_calc_updown(lora, module_k, self.in_proj_weight)
+                updown_v = lora_calc_updown(lora, module_v, self.in_proj_weight)
+                updown_qkv = torch.vstack([updown_q, updown_k, updown_v])
+
+                self.in_proj_weight += updown_qkv
+                self.out_proj.weight += lora_calc_updown(lora, module_out, self.out_proj.weight)
+                continue
+
+            if module is None:
+                continue
+
+            print(f'failed to calculate lora weights for layer {lora_layer_name}')
+
+        self.lora_current_names = wanted_names
+
+
+def lora_forward(module, input, original_forward):
+    """
+    Old way of applying Lora by executing operations during layer's forward.
+    Stacking many loras this way results in big performance degradation.
+    """
 
-def lora_forward(module, input, res):
     if len(loaded_loras) == 0:
-        return res
+        return original_forward(module, input)
+
+    input = devices.cond_cast_unet(input)
+
+    lora_restore_weights_from_backup(module)
+    lora_reset_cached_weight(module)
+
+    res = original_forward(module, input)
 
     lora_layer_name = getattr(module, 'lora_layer_name', None)
     for lora in loaded_loras:
         module = lora.modules.get(lora_layer_name, None)
-        if module is not None:
-            res = res + module.up(module.down(input)) * lora.multiplier * (module.alpha / module.up.weight.shape[1] if module.alpha else 1.0)
+        if module is None:
+            continue
+
+        module.up.to(device=devices.device)
+        module.down.to(device=devices.device)
+
+        res = res + module.up(module.down(input)) * lora.multiplier * (module.alpha / module.up.weight.shape[1] if module.alpha else 1.0)
 
     return res
 
 
+def lora_reset_cached_weight(self: Union[torch.nn.Conv2d, torch.nn.Linear]):
+    self.lora_current_names = ()
+    self.lora_weights_backup = None
+
+
 def lora_Linear_forward(self, input):
-    return lora_forward(self, input, torch.nn.Linear_forward_before_lora(self, input))
+    if shared.opts.lora_functional:
+        return lora_forward(self, input, torch.nn.Linear_forward_before_lora)
+
+    lora_apply_weights(self)
+
+    return torch.nn.Linear_forward_before_lora(self, input)
+
+
+def lora_Linear_load_state_dict(self, *args, **kwargs):
+    lora_reset_cached_weight(self)
+
+    return torch.nn.Linear_load_state_dict_before_lora(self, *args, **kwargs)
 
 
 def lora_Conv2d_forward(self, input):
-    return lora_forward(self, input, torch.nn.Conv2d_forward_before_lora(self, input))
+    if shared.opts.lora_functional:
+        return lora_forward(self, input, torch.nn.Conv2d_forward_before_lora)
+
+    lora_apply_weights(self)
+
+    return torch.nn.Conv2d_forward_before_lora(self, input)
+
+
+def lora_Conv2d_load_state_dict(self, *args, **kwargs):
+    lora_reset_cached_weight(self)
+
+    return torch.nn.Conv2d_load_state_dict_before_lora(self, *args, **kwargs)
+
+
+def lora_MultiheadAttention_forward(self, *args, **kwargs):
+    lora_apply_weights(self)
+
+    return torch.nn.MultiheadAttention_forward_before_lora(self, *args, **kwargs)
+
+
+def lora_MultiheadAttention_load_state_dict(self, *args, **kwargs):
+    lora_reset_cached_weight(self)
+
+    return torch.nn.MultiheadAttention_load_state_dict_before_lora(self, *args, **kwargs)
 
 
 def list_available_loras():
     available_loras.clear()
+    available_lora_aliases.clear()
+    forbidden_lora_aliases.clear()
+    available_lora_hash_lookup.clear()
+    forbidden_lora_aliases.update({"none": 1, "Addams": 1})
 
     os.makedirs(shared.cmd_opts.lora_dir, exist_ok=True)
 
-    candidates = \
-        glob.glob(os.path.join(shared.cmd_opts.lora_dir, '**/*.pt'), recursive=True) + \
-        glob.glob(os.path.join(shared.cmd_opts.lora_dir, '**/*.safetensors'), recursive=True) + \
-        glob.glob(os.path.join(shared.cmd_opts.lora_dir, '**/*.ckpt'), recursive=True)
-
-    for filename in sorted(candidates):
+    candidates = list(shared.walk_files(shared.cmd_opts.lora_dir, allowed_extensions=[".pt", ".ckpt", ".safetensors"]))
+    for filename in sorted(candidates, key=str.lower):
         if os.path.isdir(filename):
             continue
 
         name = os.path.splitext(os.path.basename(filename))[0]
+        try:
+            entry = LoraOnDisk(name, filename)
+        except OSError:  # should catch FileNotFoundError and PermissionError etc.
+            errors.report(f"Failed to load LoRA {name} from {filename}", exc_info=True)
+            continue
 
-        available_loras[name] = LoraOnDisk(name, filename)
+        available_loras[name] = entry
+
+        if entry.alias in available_lora_aliases:
+            forbidden_lora_aliases[entry.alias.lower()] = 1
+
+        available_lora_aliases[name] = entry
+        available_lora_aliases[entry.alias] = entry
+
+
+re_lora_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)")
+
+
+def infotext_pasted(infotext, params):
+    if "AddNet Module 1" in [x[1] for x in scripts.scripts_txt2img.infotext_fields]:
+        return  # if the other extension is active, it will handle those fields, no need to do anything
+
+    added = []
+
+    for k in params:
+        if not k.startswith("AddNet Model "):
+            continue
+
+        num = k[13:]
+
+        if params.get("AddNet Module " + num) != "LoRA":
+            continue
+
+        name = params.get("AddNet Model " + num)
+        if name is None:
+            continue
+
+        m = re_lora_name.match(name)
+        if m:
+            name = m.group(1)
+
+        multiplier = params.get("AddNet Weight A " + num, "1.0")
+
+        added.append(f"<lora:{name}:{multiplier}>")
+
+    if added:
+        params["Prompt"] += "\n" + "".join(added)
 
 
 available_loras = {}
+available_lora_aliases = {}
+available_lora_hash_lookup = {}
+forbidden_lora_aliases = {}
 loaded_loras = []
 
 list_available_loras()

extensions-builtin/Lora/scripts/lora_script.py

@@ -1,14 +1,21 @@
+import re
+
 import torch
+import gradio as gr
+from fastapi import FastAPI
 
 import lora
 import extra_networks_lora
 import ui_extra_networks_lora
-from modules import script_callbacks, ui_extra_networks, extra_networks
-
+from modules import script_callbacks, ui_extra_networks, extra_networks, shared
 
 def unload():
     torch.nn.Linear.forward = torch.nn.Linear_forward_before_lora
+    torch.nn.Linear._load_from_state_dict = torch.nn.Linear_load_state_dict_before_lora
     torch.nn.Conv2d.forward = torch.nn.Conv2d_forward_before_lora
+    torch.nn.Conv2d._load_from_state_dict = torch.nn.Conv2d_load_state_dict_before_lora
+    torch.nn.MultiheadAttention.forward = torch.nn.MultiheadAttention_forward_before_lora
+    torch.nn.MultiheadAttention._load_from_state_dict = torch.nn.MultiheadAttention_load_state_dict_before_lora
 
 
 def before_ui():
@@ -19,12 +26,91 @@ def before_ui():
 if not hasattr(torch.nn, 'Linear_forward_before_lora'):
     torch.nn.Linear_forward_before_lora = torch.nn.Linear.forward
 
+if not hasattr(torch.nn, 'Linear_load_state_dict_before_lora'):
+    torch.nn.Linear_load_state_dict_before_lora = torch.nn.Linear._load_from_state_dict
+
 if not hasattr(torch.nn, 'Conv2d_forward_before_lora'):
     torch.nn.Conv2d_forward_before_lora = torch.nn.Conv2d.forward
 
+if not hasattr(torch.nn, 'Conv2d_load_state_dict_before_lora'):
+    torch.nn.Conv2d_load_state_dict_before_lora = torch.nn.Conv2d._load_from_state_dict
+
+if not hasattr(torch.nn, 'MultiheadAttention_forward_before_lora'):
+    torch.nn.MultiheadAttention_forward_before_lora = torch.nn.MultiheadAttention.forward
+
+if not hasattr(torch.nn, 'MultiheadAttention_load_state_dict_before_lora'):
+    torch.nn.MultiheadAttention_load_state_dict_before_lora = torch.nn.MultiheadAttention._load_from_state_dict
+
 torch.nn.Linear.forward = lora.lora_Linear_forward
+torch.nn.Linear._load_from_state_dict = lora.lora_Linear_load_state_dict
 torch.nn.Conv2d.forward = lora.lora_Conv2d_forward
+torch.nn.Conv2d._load_from_state_dict = lora.lora_Conv2d_load_state_dict
+torch.nn.MultiheadAttention.forward = lora.lora_MultiheadAttention_forward
+torch.nn.MultiheadAttention._load_from_state_dict = lora.lora_MultiheadAttention_load_state_dict
 
 script_callbacks.on_model_loaded(lora.assign_lora_names_to_compvis_modules)
 script_callbacks.on_script_unloaded(unload)
 script_callbacks.on_before_ui(before_ui)
+script_callbacks.on_infotext_pasted(lora.infotext_pasted)
+
+
+shared.options_templates.update(shared.options_section(('extra_networks', "Extra Networks"), {
+    "sd_lora": shared.OptionInfo("None", "Add Lora to prompt", gr.Dropdown, lambda: {"choices": ["None", *lora.available_loras]}, refresh=lora.list_available_loras),
+    "lora_preferred_name": shared.OptionInfo("Alias from file", "When adding to prompt, refer to Lora by", gr.Radio, {"choices": ["Alias from file", "Filename"]}),
+    "lora_add_hashes_to_infotext": shared.OptionInfo(True, "Add Lora hashes to infotext"),
+}))
+
+
+shared.options_templates.update(shared.options_section(('compatibility', "Compatibility"), {
+    "lora_functional": shared.OptionInfo(False, "Lora: use old method that takes longer when you have multiple Loras active and produces same results as kohya-ss/sd-webui-additional-networks extension"),
+}))
+
+
+def create_lora_json(obj: lora.LoraOnDisk):
+    return {
+        "name": obj.name,
+        "alias": obj.alias,
+        "path": obj.filename,
+        "metadata": obj.metadata,
+    }
+
+
+def api_loras(_: gr.Blocks, app: FastAPI):
+    @app.get("/sdapi/v1/loras")
+    async def get_loras():
+        return [create_lora_json(obj) for obj in lora.available_loras.values()]
+
+    @app.post("/sdapi/v1/refresh-loras")
+    async def refresh_loras():
+        return lora.list_available_loras()
+
+
+script_callbacks.on_app_started(api_loras)
+
+re_lora = re.compile("<lora:([^:]+):")
+
+
+def infotext_pasted(infotext, d):
+    hashes = d.get("Lora hashes")
+    if not hashes:
+        return
+
+    hashes = [x.strip().split(':', 1) for x in hashes.split(",")]
+    hashes = {x[0].strip().replace(",", ""): x[1].strip() for x in hashes}
+
+    def lora_replacement(m):
+        alias = m.group(1)
+        shorthash = hashes.get(alias)
+        if shorthash is None:
+            return m.group(0)
+
+        lora_on_disk = lora.available_lora_hash_lookup.get(shorthash)
+        if lora_on_disk is None:
+            return m.group(0)
+
+        return f'<lora:{lora_on_disk.get_alias()}:'
+
+    d["Prompt"] = re.sub(re_lora, lora_replacement, d["Prompt"])
+
+
+script_callbacks.on_infotext_pasted(infotext_pasted)

extensions-builtin/Lora/ui_extra_networks_lora.py

@@ -13,22 +13,22 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage):
         lora.list_available_loras()
 
     def list_items(self):
-        for name, lora_on_disk in lora.available_loras.items():
+        for index, (name, lora_on_disk) in enumerate(lora.available_loras.items()):
             path, ext = os.path.splitext(lora_on_disk.filename)
-            previews = [path + ".png", path + ".preview.png"]
 
-            preview = None
-            for file in previews:
-                if os.path.isfile(file):
-                    preview = "./file=" + file.replace('\\', '/') + "?mtime=" + str(os.path.getmtime(file))
-                    break
+            alias = lora_on_disk.get_alias()
 
             yield {
                 "name": name,
                 "filename": path,
-                "preview": preview,
-                "prompt": json.dumps(f"<lora:{name}:") + " + opts.extra_networks_default_multiplier + " + json.dumps(">"),
-                "local_preview": path + ".png",
+                "preview": self.find_preview(path),
+                "description": self.find_description(path),
+                "search_term": self.search_terms_from_path(lora_on_disk.filename),
+                "prompt": json.dumps(f"<lora:{alias}:") + " + opts.extra_networks_default_multiplier + " + json.dumps(">"),
+                "local_preview": f"{path}.{shared.opts.samples_format}",
+                "metadata": json.dumps(lora_on_disk.metadata, indent=4) if lora_on_disk.metadata else None,
+                "sort_keys": {'default': index, **self.get_sort_keys(lora_on_disk.filename)},
+
             }
 
     def allowed_directories_for_previews(self):

extensions-builtin/ScuNET/scripts/scunet_model.py

@@ -1,16 +1,19 @@
 import os.path
 import sys
-import traceback
 
 import PIL.Image
 import numpy as np
 import torch
+from tqdm import tqdm
+
 from basicsr.utils.download_util import load_file_from_url
 
 import modules.upscaler
-from modules import devices, modelloader
+from modules import devices, modelloader, script_callbacks, errors
 from scunet_model_arch import SCUNet as net
 
+from modules.shared import opts
+
 
 class UpscalerScuNET(modules.upscaler.Upscaler):
     def __init__(self, dirname):
@@ -35,41 +38,89 @@ class UpscalerScuNET(modules.upscaler.Upscaler):
                 scaler_data = modules.upscaler.UpscalerData(name, file, self, 4)
                 scalers.append(scaler_data)
             except Exception:
-                print(f"Error loading ScuNET model: {file}", file=sys.stderr)
-                print(traceback.format_exc(), file=sys.stderr)
+                errors.report(f"Error loading ScuNET model: {file}", exc_info=True)
         if add_model2:
             scaler_data2 = modules.upscaler.UpscalerData(self.model_name2, self.model_url2, self)
             scalers.append(scaler_data2)
         self.scalers = scalers
 
-    def do_upscale(self, img: PIL.Image, selected_file):
+    @staticmethod
+    @torch.no_grad()
+    def tiled_inference(img, model):
+        # test the image tile by tile
+        h, w = img.shape[2:]
+        tile = opts.SCUNET_tile
+        tile_overlap = opts.SCUNET_tile_overlap
+        if tile == 0:
+            return model(img)
+
+        device = devices.get_device_for('scunet')
+        assert tile % 8 == 0, "tile size should be a multiple of window_size"
+        sf = 1
+
+        stride = tile - tile_overlap
+        h_idx_list = list(range(0, h - tile, stride)) + [h - tile]
+        w_idx_list = list(range(0, w - tile, stride)) + [w - tile]
+        E = torch.zeros(1, 3, h * sf, w * sf, dtype=img.dtype, device=device)
+        W = torch.zeros_like(E, dtype=devices.dtype, device=device)
+
+        with tqdm(total=len(h_idx_list) * len(w_idx_list), desc="ScuNET tiles") as pbar:
+            for h_idx in h_idx_list:
+
+                for w_idx in w_idx_list:
+
+                    in_patch = img[..., h_idx: h_idx + tile, w_idx: w_idx + tile]
+
+                    out_patch = model(in_patch)
+                    out_patch_mask = torch.ones_like(out_patch)
+
+                    E[
+                        ..., h_idx * sf: (h_idx + tile) * sf, w_idx * sf: (w_idx + tile) * sf
+                    ].add_(out_patch)
+                    W[
+                        ..., h_idx * sf: (h_idx + tile) * sf, w_idx * sf: (w_idx + tile) * sf
+                    ].add_(out_patch_mask)
+                    pbar.update(1)
+        output = E.div_(W)
+
+        return output
+
+    def do_upscale(self, img: PIL.Image.Image, selected_file):
+
         torch.cuda.empty_cache()
 
         model = self.load_model(selected_file)
         if model is None:
+            print(f"ScuNET: Unable to load model from {selected_file}", file=sys.stderr)
             return img
 
         device = devices.get_device_for('scunet')
-        img = np.array(img)
-        img = img[:, :, ::-1]
-        img = np.moveaxis(img, 2, 0) / 255
-        img = torch.from_numpy(img).float()
-        img = img.unsqueeze(0).to(device)
+        tile = opts.SCUNET_tile
+        h, w = img.height, img.width
+        np_img = np.array(img)
+        np_img = np_img[:, :, ::-1]  # RGB to BGR
+        np_img = np_img.transpose((2, 0, 1)) / 255  # HWC to CHW
+        torch_img = torch.from_numpy(np_img).float().unsqueeze(0).to(device)  # type: ignore
 
-        with torch.no_grad():
-            output = model(img)
-        output = output.squeeze().float().cpu().clamp_(0, 1).numpy()
-        output = 255. * np.moveaxis(output, 0, 2)
-        output = output.astype(np.uint8)
-        output = output[:, :, ::-1]
+        if tile > h or tile > w:
+            _img = torch.zeros(1, 3, max(h, tile), max(w, tile), dtype=torch_img.dtype, device=torch_img.device)
+            _img[:, :, :h, :w] = torch_img # pad image
+            torch_img = _img
+
+        torch_output = self.tiled_inference(torch_img, model).squeeze(0)
+        torch_output = torch_output[:, :h * 1, :w * 1] # remove padding, if any
+        np_output: np.ndarray = torch_output.float().cpu().clamp_(0, 1).numpy()
+        del torch_img, torch_output
         torch.cuda.empty_cache()
-        return PIL.Image.fromarray(output, 'RGB')
+
+        output = np_output.transpose((1, 2, 0))  # CHW to HWC
+        output = output[:, :, ::-1]  # BGR to RGB
+        return PIL.Image.fromarray((output * 255).astype(np.uint8))
 
     def load_model(self, path: str):
         device = devices.get_device_for('scunet')
         if "http" in path:
-            filename = load_file_from_url(url=self.model_url, model_dir=self.model_path, file_name="%s.pth" % self.name,
-                                          progress=True)
+            filename = load_file_from_url(url=self.model_url, model_dir=self.model_download_path, file_name="%s.pth" % self.name, progress=True)
         else:
             filename = path
         if not os.path.exists(os.path.join(self.model_path, filename)) or filename is None:
@@ -79,9 +130,19 @@ class UpscalerScuNET(modules.upscaler.Upscaler):
         model = net(in_nc=3, config=[4, 4, 4, 4, 4, 4, 4], dim=64)
         model.load_state_dict(torch.load(filename), strict=True)
         model.eval()
-        for k, v in model.named_parameters():
+        for _, v in model.named_parameters():
             v.requires_grad = False
         model = model.to(device)
 
         return model
 
+
+def on_ui_settings():
+    import gradio as gr
+    from modules import shared
+
+    shared.opts.add_option("SCUNET_tile", shared.OptionInfo(256, "Tile size for SCUNET upscalers.", gr.Slider, {"minimum": 0, "maximum": 512, "step": 16}, section=('upscaling', "Upscaling")).info("0 = no tiling"))
+    shared.opts.add_option("SCUNET_tile_overlap", shared.OptionInfo(8, "Tile overlap for SCUNET upscalers.", gr.Slider, {"minimum": 0, "maximum": 64, "step": 1}, section=('upscaling', "Upscaling")).info("Low values = visible seam"))
+
+
+script_callbacks.on_ui_settings(on_ui_settings)

extensions-builtin/ScuNET/scunet_model_arch.py

@@ -61,7 +61,9 @@ class WMSA(nn.Module):
         Returns:
             output: tensor shape [b h w c]
         """
-        if self.type != 'W': x = torch.roll(x, shifts=(-(self.window_size // 2), -(self.window_size // 2)), dims=(1, 2))
+        if self.type != 'W':
+            x = torch.roll(x, shifts=(-(self.window_size // 2), -(self.window_size // 2)), dims=(1, 2))
+
         x = rearrange(x, 'b (w1 p1) (w2 p2) c -> b w1 w2 p1 p2 c', p1=self.window_size, p2=self.window_size)
         h_windows = x.size(1)
         w_windows = x.size(2)
@@ -85,8 +87,9 @@ class WMSA(nn.Module):
         output = self.linear(output)
         output = rearrange(output, 'b (w1 w2) (p1 p2) c -> b (w1 p1) (w2 p2) c', w1=h_windows, p1=self.window_size)
 
-        if self.type != 'W': output = torch.roll(output, shifts=(self.window_size // 2, self.window_size // 2),
-                                                 dims=(1, 2))
+        if self.type != 'W':
+            output = torch.roll(output, shifts=(self.window_size // 2, self.window_size // 2), dims=(1, 2))
+
         return output
 
     def relative_embedding(self):
@@ -262,4 +265,4 @@ class SCUNet(nn.Module):
                 nn.init.constant_(m.bias, 0)
         elif isinstance(m, nn.LayerNorm):
             nn.init.constant_(m.bias, 0)
-            nn.init.constant_(m.weight, 1.0)
+            nn.init.constant_(m.weight, 1.0)

extensions-builtin/SwinIR/scripts/swinir_model.py

@@ -1,4 +1,3 @@
-import contextlib
 import os
 
 import numpy as np
@@ -8,7 +7,7 @@ from basicsr.utils.download_util import load_file_from_url
 from tqdm import tqdm
 
 from modules import modelloader, devices, script_callbacks, shared
-from modules.shared import cmd_opts, opts, state
+from modules.shared import opts, state
 from swinir_model_arch import SwinIR as net
 from swinir_model_arch_v2 import Swin2SR as net2
 from modules.upscaler import Upscaler, UpscalerData
@@ -45,14 +44,14 @@ class UpscalerSwinIR(Upscaler):
         img = upscale(img, model)
         try:
             torch.cuda.empty_cache()
-        except:
+        except Exception:
             pass
         return img
 
     def load_model(self, path, scale=4):
         if "http" in path:
             dl_name = "%s%s" % (self.model_name.replace(" ", "_"), ".pth")
-            filename = load_file_from_url(url=path, model_dir=self.model_path, file_name=dl_name, progress=True)
+            filename = load_file_from_url(url=path, model_dir=self.model_download_path, file_name=dl_name, progress=True)
         else:
             filename = path
         if filename is None or not os.path.exists(filename):
@@ -151,7 +150,7 @@ def inference(img, model, tile, tile_overlap, window_size, scale):
             for w_idx in w_idx_list:
                 if state.interrupted or state.skipped:
                     break
-                
+
                 in_patch = img[..., h_idx: h_idx + tile, w_idx: w_idx + tile]
                 out_patch = model(in_patch)
                 out_patch_mask = torch.ones_like(out_patch)

extensions-builtin/SwinIR/swinir_model_arch.py

@@ -644,7 +644,7 @@ class SwinIR(nn.Module):
     """
 
     def __init__(self, img_size=64, patch_size=1, in_chans=3,
-                 embed_dim=96, depths=[6, 6, 6, 6], num_heads=[6, 6, 6, 6],
+                 embed_dim=96, depths=(6, 6, 6, 6), num_heads=(6, 6, 6, 6),
                  window_size=7, mlp_ratio=4., qkv_bias=True, qk_scale=None,
                  drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,
                  norm_layer=nn.LayerNorm, ape=False, patch_norm=True,
@@ -805,7 +805,7 @@ class SwinIR(nn.Module):
     def forward(self, x):
         H, W = x.shape[2:]
         x = self.check_image_size(x)
-        
+
         self.mean = self.mean.type_as(x)
         x = (x - self.mean) * self.img_range
 
@@ -844,7 +844,7 @@ class SwinIR(nn.Module):
         H, W = self.patches_resolution
         flops += H * W * 3 * self.embed_dim * 9
         flops += self.patch_embed.flops()
-        for i, layer in enumerate(self.layers):
+        for layer in self.layers:
             flops += layer.flops()
         flops += H * W * 3 * self.embed_dim * self.embed_dim
         flops += self.upsample.flops()

extensions-builtin/SwinIR/swinir_model_arch_v2.py

@@ -74,7 +74,7 @@ class WindowAttention(nn.Module):
     """
 
     def __init__(self, dim, window_size, num_heads, qkv_bias=True, attn_drop=0., proj_drop=0.,
-                 pretrained_window_size=[0, 0]):
+                 pretrained_window_size=(0, 0)):
 
         super().__init__()
         self.dim = dim
@@ -241,7 +241,7 @@ class SwinTransformerBlock(nn.Module):
             attn_mask = None
 
         self.register_buffer("attn_mask", attn_mask)
-        
+
     def calculate_mask(self, x_size):
         # calculate attention mask for SW-MSA
         H, W = x_size
@@ -263,7 +263,7 @@ class SwinTransformerBlock(nn.Module):
         attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
         attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
 
-        return attn_mask        
+        return attn_mask
 
     def forward(self, x, x_size):
         H, W = x_size
@@ -288,7 +288,7 @@ class SwinTransformerBlock(nn.Module):
             attn_windows = self.attn(x_windows, mask=self.attn_mask)  # nW*B, window_size*window_size, C
         else:
             attn_windows = self.attn(x_windows, mask=self.calculate_mask(x_size).to(x.device))
-            
+
         # merge windows
         attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C)
         shifted_x = window_reverse(attn_windows, self.window_size, H, W)  # B H' W' C
@@ -369,7 +369,7 @@ class PatchMerging(nn.Module):
         H, W = self.input_resolution
         flops = (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim
         flops += H * W * self.dim // 2
-        return flops    
+        return flops
 
 class BasicLayer(nn.Module):
     """ A basic Swin Transformer layer for one stage.
@@ -447,7 +447,7 @@ class BasicLayer(nn.Module):
             nn.init.constant_(blk.norm1.weight, 0)
             nn.init.constant_(blk.norm2.bias, 0)
             nn.init.constant_(blk.norm2.weight, 0)
-            
+
 class PatchEmbed(nn.Module):
     r""" Image to Patch Embedding
     Args:
@@ -492,7 +492,7 @@ class PatchEmbed(nn.Module):
         flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
         if self.norm is not None:
             flops += Ho * Wo * self.embed_dim
-        return flops           
+        return flops
 
 class RSTB(nn.Module):
     """Residual Swin Transformer Block (RSTB).
@@ -531,7 +531,7 @@ class RSTB(nn.Module):
                                          num_heads=num_heads,
                                          window_size=window_size,
                                          mlp_ratio=mlp_ratio,
-                                         qkv_bias=qkv_bias, 
+                                         qkv_bias=qkv_bias,
                                          drop=drop, attn_drop=attn_drop,
                                          drop_path=drop_path,
                                          norm_layer=norm_layer,
@@ -622,7 +622,7 @@ class Upsample(nn.Sequential):
         else:
             raise ValueError(f'scale {scale} is not supported. ' 'Supported scales: 2^n and 3.')
         super(Upsample, self).__init__(*m)
-        
+
 class Upsample_hf(nn.Sequential):
     """Upsample module.
 
@@ -642,7 +642,7 @@ class Upsample_hf(nn.Sequential):
             m.append(nn.PixelShuffle(3))
         else:
             raise ValueError(f'scale {scale} is not supported. ' 'Supported scales: 2^n and 3.')
-        super(Upsample_hf, self).__init__(*m)        
+        super(Upsample_hf, self).__init__(*m)
 
 
 class UpsampleOneStep(nn.Sequential):
@@ -667,8 +667,8 @@ class UpsampleOneStep(nn.Sequential):
         H, W = self.input_resolution
         flops = H * W * self.num_feat * 3 * 9
         return flops
-    
-    
+
+
 
 class Swin2SR(nn.Module):
     r""" Swin2SR
@@ -698,8 +698,8 @@ class Swin2SR(nn.Module):
     """
 
     def __init__(self, img_size=64, patch_size=1, in_chans=3,
-                 embed_dim=96, depths=[6, 6, 6, 6], num_heads=[6, 6, 6, 6],
-                 window_size=7, mlp_ratio=4., qkv_bias=True, 
+                 embed_dim=96, depths=(6, 6, 6, 6), num_heads=(6, 6, 6, 6),
+                 window_size=7, mlp_ratio=4., qkv_bias=True,
                  drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1,
                  norm_layer=nn.LayerNorm, ape=False, patch_norm=True,
                  use_checkpoint=False, upscale=2, img_range=1., upsampler='', resi_connection='1conv',
@@ -764,7 +764,7 @@ class Swin2SR(nn.Module):
                          num_heads=num_heads[i_layer],
                          window_size=window_size,
                          mlp_ratio=self.mlp_ratio,
-                         qkv_bias=qkv_bias, 
+                         qkv_bias=qkv_bias,
                          drop=drop_rate, attn_drop=attn_drop_rate,
                          drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],  # no impact on SR results
                          norm_layer=norm_layer,
@@ -776,7 +776,7 @@ class Swin2SR(nn.Module):
 
                          )
             self.layers.append(layer)
-            
+
         if self.upsampler == 'pixelshuffle_hf':
             self.layers_hf = nn.ModuleList()
             for i_layer in range(self.num_layers):
@@ -787,7 +787,7 @@ class Swin2SR(nn.Module):
                              num_heads=num_heads[i_layer],
                              window_size=window_size,
                              mlp_ratio=self.mlp_ratio,
-                             qkv_bias=qkv_bias, 
+                             qkv_bias=qkv_bias,
                              drop=drop_rate, attn_drop=attn_drop_rate,
                              drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],  # no impact on SR results
                              norm_layer=norm_layer,
@@ -799,7 +799,7 @@ class Swin2SR(nn.Module):
 
                              )
                 self.layers_hf.append(layer)
-                        
+
         self.norm = norm_layer(self.num_features)
 
         # build the last conv layer in deep feature extraction
@@ -829,10 +829,10 @@ class Swin2SR(nn.Module):
             self.conv_aux = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1)
             self.conv_after_aux = nn.Sequential(
                 nn.Conv2d(3, num_feat, 3, 1, 1),
-                nn.LeakyReLU(inplace=True))            
+                nn.LeakyReLU(inplace=True))
             self.upsample = Upsample(upscale, num_feat)
             self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1)
-            
+
         elif self.upsampler == 'pixelshuffle_hf':
             self.conv_before_upsample = nn.Sequential(nn.Conv2d(embed_dim, num_feat, 3, 1, 1),
                                                       nn.LeakyReLU(inplace=True))
@@ -846,7 +846,7 @@ class Swin2SR(nn.Module):
                 nn.Conv2d(embed_dim, num_feat, 3, 1, 1),
                 nn.LeakyReLU(inplace=True))
             self.conv_last_hf = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1)
-            
+
         elif self.upsampler == 'pixelshuffledirect':
             # for lightweight SR (to save parameters)
             self.upsample = UpsampleOneStep(upscale, embed_dim, num_out_ch,
@@ -905,7 +905,7 @@ class Swin2SR(nn.Module):
         x = self.patch_unembed(x, x_size)
 
         return x
-    
+
     def forward_features_hf(self, x):
         x_size = (x.shape[2], x.shape[3])
         x = self.patch_embed(x)
@@ -919,7 +919,7 @@ class Swin2SR(nn.Module):
         x = self.norm(x)  # B L C
         x = self.patch_unembed(x, x_size)
 
-        return x    
+        return x
 
     def forward(self, x):
         H, W = x.shape[2:]
@@ -951,7 +951,7 @@ class Swin2SR(nn.Module):
             x = self.conv_after_body(self.forward_features(x)) + x
             x_before = self.conv_before_upsample(x)
             x_out = self.conv_last(self.upsample(x_before))
-            
+
             x_hf = self.conv_first_hf(x_before)
             x_hf = self.conv_after_body_hf(self.forward_features_hf(x_hf)) + x_hf
             x_hf = self.conv_before_upsample_hf(x_hf)
@@ -977,15 +977,15 @@ class Swin2SR(nn.Module):
             x_first = self.conv_first(x)
             res = self.conv_after_body(self.forward_features(x_first)) + x_first
             x = x + self.conv_last(res)
-        
+
         x = x / self.img_range + self.mean
         if self.upsampler == "pixelshuffle_aux":
             return x[:, :, :H*self.upscale, :W*self.upscale], aux
-        
+
         elif self.upsampler == "pixelshuffle_hf":
             x_out = x_out / self.img_range + self.mean
             return x_out[:, :, :H*self.upscale, :W*self.upscale], x[:, :, :H*self.upscale, :W*self.upscale], x_hf[:, :, :H*self.upscale, :W*self.upscale]
-        
+
         else:
             return x[:, :, :H*self.upscale, :W*self.upscale]
 
@@ -994,7 +994,7 @@ class Swin2SR(nn.Module):
         H, W = self.patches_resolution
         flops += H * W * 3 * self.embed_dim * 9
         flops += self.patch_embed.flops()
-        for i, layer in enumerate(self.layers):
+        for layer in self.layers:
             flops += layer.flops()
         flops += H * W * 3 * self.embed_dim * self.embed_dim
         flops += self.upsample.flops()
@@ -1014,4 +1014,4 @@ if __name__ == '__main__':
 
     x = torch.randn((1, 3, height, width))
     x = model(x)
-    print(x.shape)
+    print(x.shape)

extensions-builtin/canvas-zoom-and-pan/javascript/zoom.js

@@ -0,0 +1,640 @@
+onUiLoaded(async() => {
+    const elementIDs = {
+        img2imgTabs: "#mode_img2img .tab-nav",
+        inpaint: "#img2maskimg",
+        inpaintSketch: "#inpaint_sketch",
+        rangeGroup: "#img2img_column_size",
+        sketch: "#img2img_sketch",
+    };
+    const tabNameToElementId = {
+        "Inpaint sketch": elementIDs.inpaintSketch,
+        "Inpaint": elementIDs.inpaint,
+        "Sketch": elementIDs.sketch,
+    };
+
+    // Helper functions
+    // Get active tab
+    function getActiveTab(elements, all = false) {
+        const tabs = elements.img2imgTabs.querySelectorAll("button");
+
+        if (all) return tabs;
+
+        for (let tab of tabs) {
+            if (tab.classList.contains("selected")) {
+                return tab;
+            }
+        }
+    }
+
+    // Get tab ID
+    function getTabId(elements) {
+        const activeTab = getActiveTab(elements);
+        return tabNameToElementId[activeTab.innerText];
+    }
+
+    // Wait until opts loaded
+    async function waitForOpts() {
+        for (;;) {
+            if (window.opts && Object.keys(window.opts).length) {
+                return window.opts;
+            }
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+    }
+
+    // Check is hotkey valid
+    function isSingleLetter(value) {
+        return (
+            typeof value === "string" && value.length === 1 && /[a-z]/i.test(value)
+        );
+    }
+
+    // Create hotkeyConfig from opts
+    function createHotkeyConfig(defaultHotkeysConfig, hotkeysConfigOpts) {
+        const result = {};
+        const usedKeys = new Set();
+
+        for (const key in defaultHotkeysConfig) {
+            if (typeof hotkeysConfigOpts[key] === "boolean") {
+                result[key] = hotkeysConfigOpts[key];
+                continue;
+            }
+            if (
+                hotkeysConfigOpts[key] &&
+                isSingleLetter(hotkeysConfigOpts[key]) &&
+                !usedKeys.has(hotkeysConfigOpts[key].toUpperCase())
+            ) {
+                // If the property passed the test and has not yet been used, add 'Key' before it and save it
+                result[key] = "Key" + hotkeysConfigOpts[key].toUpperCase();
+                usedKeys.add(hotkeysConfigOpts[key].toUpperCase());
+            } else {
+                // If the property does not pass the test or has already been used, we keep the default value
+                console.error(
+                    `Hotkey: ${hotkeysConfigOpts[key]} for ${key} is repeated and conflicts with another hotkey or is not 1 letter. The default hotkey is used: ${defaultHotkeysConfig[key][3]}`
+                );
+                result[key] = defaultHotkeysConfig[key];
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * The restoreImgRedMask function displays a red mask around an image to indicate the aspect ratio.
+     * If the image display property is set to 'none', the mask breaks. To fix this, the function
+     * temporarily sets the display property to 'block' and then hides the mask again after 300 milliseconds
+     * to avoid breaking the canvas. Additionally, the function adjusts the mask to work correctly on
+     * very long images.
+     */
+    function restoreImgRedMask(elements) {
+        const mainTabId = getTabId(elements);
+
+        if (!mainTabId) return;
+
+        const mainTab = gradioApp().querySelector(mainTabId);
+        const img = mainTab.querySelector("img");
+        const imageARPreview = gradioApp().querySelector("#imageARPreview");
+
+        if (!img || !imageARPreview) return;
+
+        imageARPreview.style.transform = "";
+        if (parseFloat(mainTab.style.width) > 865) {
+            const transformString = mainTab.style.transform;
+            const scaleMatch = transformString.match(/scale\(([-+]?[0-9]*\.?[0-9]+)\)/);
+            let zoom = 1; // default zoom
+
+            if (scaleMatch && scaleMatch[1]) {
+                zoom = Number(scaleMatch[1]);
+            }
+
+            imageARPreview.style.transformOrigin = "0 0";
+            imageARPreview.style.transform = `scale(${zoom})`;
+        }
+
+        if (img.style.display !== "none") return;
+
+        img.style.display = "block";
+
+        setTimeout(() => {
+            img.style.display = "none";
+        }, 400);
+    }
+
+    const hotkeysConfigOpts = await waitForOpts();
+
+    // Default config
+    const defaultHotkeysConfig = {
+        canvas_hotkey_reset: "KeyR",
+        canvas_hotkey_fullscreen: "KeyS",
+        canvas_hotkey_move: "KeyF",
+        canvas_hotkey_overlap: "KeyO",
+        canvas_show_tooltip: true,
+        canvas_swap_controls: false
+    };
+    // swap the actions for ctr + wheel and shift + wheel
+    const hotkeysConfig = createHotkeyConfig(
+        defaultHotkeysConfig,
+        hotkeysConfigOpts
+    );
+
+    let isMoving = false;
+    let mouseX, mouseY;
+    let activeElement;
+
+    const elements = Object.fromEntries(Object.keys(elementIDs).map((id) => [
+        id,
+        gradioApp().querySelector(elementIDs[id]),
+    ]));
+    const elemData = {};
+
+    // Apply functionality to the range inputs. Restore redmask and correct for long images.
+    const rangeInputs = elements.rangeGroup ? Array.from(elements.rangeGroup.querySelectorAll("input")) :
+        [
+            gradioApp().querySelector("#img2img_width input[type='range']"),
+            gradioApp().querySelector("#img2img_height input[type='range']")
+        ];
+
+    for (const input of rangeInputs) {
+        input?.addEventListener("input", () => restoreImgRedMask(elements));
+    }
+
+    function applyZoomAndPan(elemId) {
+        const targetElement = gradioApp().querySelector(elemId);
+
+        if (!targetElement) {
+            console.log("Element not found");
+            return;
+        }
+
+        targetElement.style.transformOrigin = "0 0";
+
+        elemData[elemId] = {
+            zoom: 1,
+            panX: 0,
+            panY: 0
+        };
+        let fullScreenMode = false;
+
+        // Create tooltip
+        function createTooltip() {
+            const toolTipElemnt =
+                targetElement.querySelector(".image-container");
+            const tooltip = document.createElement("div");
+            tooltip.className = "tooltip";
+
+            // Creating an item of information
+            const info = document.createElement("i");
+            info.className = "tooltip-info";
+            info.textContent = "";
+
+            // Create a container for the contents of the tooltip
+            const tooltipContent = document.createElement("div");
+            tooltipContent.className = "tooltip-content";
+
+            // Add info about hotkeys
+            const zoomKey = hotkeysConfig.canvas_swap_controls ? "Ctrl" : "Shift";
+            const adjustKey = hotkeysConfig.canvas_swap_controls ? "Shift" : "Ctrl";
+
+            const hotkeys = [
+                {key: `${zoomKey} + wheel`, action: "Zoom canvas"},
+                {key: `${adjustKey} + wheel`, action: "Adjust brush size"},
+                {
+                    key: hotkeysConfig.canvas_hotkey_reset.charAt(hotkeysConfig.canvas_hotkey_reset.length - 1),
+                    action: "Reset zoom"
+                },
+                {
+                    key: hotkeysConfig.canvas_hotkey_fullscreen.charAt(hotkeysConfig.canvas_hotkey_fullscreen.length - 1),
+                    action: "Fullscreen mode"
+                },
+                {
+                    key: hotkeysConfig.canvas_hotkey_move.charAt(hotkeysConfig.canvas_hotkey_move.length - 1),
+                    action: "Move canvas"
+                }
+            ];
+            for (const hotkey of hotkeys) {
+                const p = document.createElement("p");
+                p.innerHTML = `<b>${hotkey.key}</b> - ${hotkey.action}`;
+                tooltipContent.appendChild(p);
+            }
+
+            // Add information and content elements to the tooltip element
+            tooltip.appendChild(info);
+            tooltip.appendChild(tooltipContent);
+
+            // Add a hint element to the target element
+            toolTipElemnt.appendChild(tooltip);
+        }
+
+        //Show tool tip if setting enable
+        if (hotkeysConfig.canvas_show_tooltip) {
+            createTooltip();
+        }
+
+        // In the course of research, it was found that the tag img is very harmful when zooming and creates white canvases. This hack allows you to almost never think about this problem, it has no effect on webui.
+        function fixCanvas() {
+            const activeTab = getActiveTab(elements).textContent.trim();
+
+            if (activeTab !== "img2img") {
+                const img = targetElement.querySelector(`${elemId} img`);
+
+                if (img && img.style.display !== "none") {
+                    img.style.display = "none";
+                    img.style.visibility = "hidden";
+                }
+            }
+        }
+
+        // Reset the zoom level and pan position of the target element to their initial values
+        function resetZoom() {
+            elemData[elemId] = {
+                zoomLevel: 1,
+                panX: 0,
+                panY: 0
+            };
+
+            fixCanvas();
+            targetElement.style.transform = `scale(${elemData[elemId].zoomLevel}) translate(${elemData[elemId].panX}px, ${elemData[elemId].panY}px)`;
+
+            const canvas = gradioApp().querySelector(
+                `${elemId} canvas[key="interface"]`
+            );
+
+            toggleOverlap("off");
+            fullScreenMode = false;
+
+            if (
+                canvas &&
+                parseFloat(canvas.style.width) > 865 &&
+                parseFloat(targetElement.style.width) > 865
+            ) {
+                fitToElement();
+                return;
+            }
+
+            targetElement.style.width = "";
+            if (canvas) {
+                targetElement.style.height = canvas.style.height;
+            }
+        }
+
+        // Toggle the zIndex of the target element between two values, allowing it to overlap or be overlapped by other elements
+        function toggleOverlap(forced = "") {
+            const zIndex1 = "0";
+            const zIndex2 = "998";
+
+            targetElement.style.zIndex =
+                targetElement.style.zIndex !== zIndex2 ? zIndex2 : zIndex1;
+
+            if (forced === "off") {
+                targetElement.style.zIndex = zIndex1;
+            } else if (forced === "on") {
+                targetElement.style.zIndex = zIndex2;
+            }
+        }
+
+        // Adjust the brush size based on the deltaY value from a mouse wheel event
+        function adjustBrushSize(
+            elemId,
+            deltaY,
+            withoutValue = false,
+            percentage = 5
+        ) {
+            const input =
+                gradioApp().querySelector(
+                    `${elemId} input[aria-label='Brush radius']`
+                ) ||
+                gradioApp().querySelector(
+                    `${elemId} button[aria-label="Use brush"]`
+                );
+
+            if (input) {
+                input.click();
+                if (!withoutValue) {
+                    const maxValue =
+                        parseFloat(input.getAttribute("max")) || 100;
+                    const changeAmount = maxValue * (percentage / 100);
+                    const newValue =
+                        parseFloat(input.value) +
+                        (deltaY > 0 ? -changeAmount : changeAmount);
+                    input.value = Math.min(Math.max(newValue, 0), maxValue);
+                    input.dispatchEvent(new Event("change"));
+                }
+            }
+        }
+
+        // Reset zoom when uploading a new image
+        const fileInput = gradioApp().querySelector(
+            `${elemId} input[type="file"][accept="image/*"].svelte-116rqfv`
+        );
+        fileInput.addEventListener("click", resetZoom);
+
+        // Update the zoom level and pan position of the target element based on the values of the zoomLevel, panX and panY variables
+        function updateZoom(newZoomLevel, mouseX, mouseY) {
+            newZoomLevel = Math.max(0.5, Math.min(newZoomLevel, 15));
+
+            elemData[elemId].panX +=
+                mouseX - (mouseX * newZoomLevel) / elemData[elemId].zoomLevel;
+            elemData[elemId].panY +=
+                mouseY - (mouseY * newZoomLevel) / elemData[elemId].zoomLevel;
+
+            targetElement.style.transformOrigin = "0 0";
+            targetElement.style.transform = `translate(${elemData[elemId].panX}px, ${elemData[elemId].panY}px) scale(${newZoomLevel})`;
+
+            toggleOverlap("on");
+            return newZoomLevel;
+        }
+
+        // Change the zoom level based on user interaction
+        function changeZoomLevel(operation, e) {
+            if (
+                (!hotkeysConfig.canvas_swap_controls && e.shiftKey) ||
+                (hotkeysConfig.canvas_swap_controls && e.ctrlKey)
+            ) {
+                e.preventDefault();
+
+                let zoomPosX, zoomPosY;
+                let delta = 0.2;
+                if (elemData[elemId].zoomLevel > 7) {
+                    delta = 0.9;
+                } else if (elemData[elemId].zoomLevel > 2) {
+                    delta = 0.6;
+                }
+
+                zoomPosX = e.clientX;
+                zoomPosY = e.clientY;
+
+                fullScreenMode = false;
+                elemData[elemId].zoomLevel = updateZoom(
+                    elemData[elemId].zoomLevel +
+                        (operation === "+" ? delta : -delta),
+                    zoomPosX - targetElement.getBoundingClientRect().left,
+                    zoomPosY - targetElement.getBoundingClientRect().top
+                );
+            }
+        }
+
+        /**
+         * This function fits the target element to the screen by calculating
+         * the required scale and offsets. It also updates the global variables
+         * zoomLevel, panX, and panY to reflect the new state.
+         */
+
+        function fitToElement() {
+            //Reset Zoom
+            targetElement.style.transform = `translate(${0}px, ${0}px) scale(${1})`;
+
+            // Get element and screen dimensions
+            const elementWidth = targetElement.offsetWidth;
+            const elementHeight = targetElement.offsetHeight;
+            const parentElement = targetElement.parentElement;
+            const screenWidth = parentElement.clientWidth;
+            const screenHeight = parentElement.clientHeight;
+
+            // Get element's coordinates relative to the parent element
+            const elementRect = targetElement.getBoundingClientRect();
+            const parentRect = parentElement.getBoundingClientRect();
+            const elementX = elementRect.x - parentRect.x;
+
+            // Calculate scale and offsets
+            const scaleX = screenWidth / elementWidth;
+            const scaleY = screenHeight / elementHeight;
+            const scale = Math.min(scaleX, scaleY);
+
+            const transformOrigin =
+                window.getComputedStyle(targetElement).transformOrigin;
+            const [originX, originY] = transformOrigin.split(" ");
+            const originXValue = parseFloat(originX);
+            const originYValue = parseFloat(originY);
+
+            const offsetX =
+                (screenWidth - elementWidth * scale) / 2 -
+                originXValue * (1 - scale);
+            const offsetY =
+                (screenHeight - elementHeight * scale) / 2.5 -
+                originYValue * (1 - scale);
+
+            // Apply scale and offsets to the element
+            targetElement.style.transform = `translate(${offsetX}px, ${offsetY}px) scale(${scale})`;
+
+            // Update global variables
+            elemData[elemId].zoomLevel = scale;
+            elemData[elemId].panX = offsetX;
+            elemData[elemId].panY = offsetY;
+
+            fullScreenMode = false;
+            toggleOverlap("off");
+        }
+
+        /**
+         * This function fits the target element to the screen by calculating
+         * the required scale and offsets. It also updates the global variables
+         * zoomLevel, panX, and panY to reflect the new state.
+         */
+
+        // Fullscreen mode
+        function fitToScreen() {
+            const canvas = gradioApp().querySelector(
+                `${elemId} canvas[key="interface"]`
+            );
+
+            if (!canvas) return;
+
+            if (canvas.offsetWidth > 862) {
+                targetElement.style.width = canvas.offsetWidth + "px";
+            }
+
+            if (fullScreenMode) {
+                resetZoom();
+                fullScreenMode = false;
+                return;
+            }
+
+            //Reset Zoom
+            targetElement.style.transform = `translate(${0}px, ${0}px) scale(${1})`;
+
+            // Get scrollbar width to right-align the image
+            const scrollbarWidth =
+                window.innerWidth - document.documentElement.clientWidth;
+
+            // Get element and screen dimensions
+            const elementWidth = targetElement.offsetWidth;
+            const elementHeight = targetElement.offsetHeight;
+            const screenWidth = window.innerWidth - scrollbarWidth;
+            const screenHeight = window.innerHeight;
+
+            // Get element's coordinates relative to the page
+            const elementRect = targetElement.getBoundingClientRect();
+            const elementY = elementRect.y;
+            const elementX = elementRect.x;
+
+            // Calculate scale and offsets
+            const scaleX = screenWidth / elementWidth;
+            const scaleY = screenHeight / elementHeight;
+            const scale = Math.min(scaleX, scaleY);
+
+            // Get the current transformOrigin
+            const computedStyle = window.getComputedStyle(targetElement);
+            const transformOrigin = computedStyle.transformOrigin;
+            const [originX, originY] = transformOrigin.split(" ");
+            const originXValue = parseFloat(originX);
+            const originYValue = parseFloat(originY);
+
+            // Calculate offsets with respect to the transformOrigin
+            const offsetX =
+                (screenWidth - elementWidth * scale) / 2 -
+                elementX -
+                originXValue * (1 - scale);
+            const offsetY =
+                (screenHeight - elementHeight * scale) / 2 -
+                elementY -
+                originYValue * (1 - scale);
+
+            // Apply scale and offsets to the element
+            targetElement.style.transform = `translate(${offsetX}px, ${offsetY}px) scale(${scale})`;
+
+            // Update global variables
+            elemData[elemId].zoomLevel = scale;
+            elemData[elemId].panX = offsetX;
+            elemData[elemId].panY = offsetY;
+
+            fullScreenMode = true;
+            toggleOverlap("on");
+        }
+
+        // Handle keydown events
+        function handleKeyDown(event) {
+            const hotkeyActions = {
+                [hotkeysConfig.canvas_hotkey_reset]: resetZoom,
+                [hotkeysConfig.canvas_hotkey_overlap]: toggleOverlap,
+                [hotkeysConfig.canvas_hotkey_fullscreen]: fitToScreen
+            };
+
+            const action = hotkeyActions[event.code];
+            if (action) {
+                event.preventDefault();
+                action(event);
+            }
+        }
+
+        // Get Mouse position
+        function getMousePosition(e) {
+            mouseX = e.offsetX;
+            mouseY = e.offsetY;
+        }
+
+        targetElement.addEventListener("mousemove", getMousePosition);
+
+        // Handle events only inside the targetElement
+        let isKeyDownHandlerAttached = false;
+
+        function handleMouseMove() {
+            if (!isKeyDownHandlerAttached) {
+                document.addEventListener("keydown", handleKeyDown);
+                isKeyDownHandlerAttached = true;
+
+                activeElement = elemId;
+            }
+        }
+
+        function handleMouseLeave() {
+            if (isKeyDownHandlerAttached) {
+                document.removeEventListener("keydown", handleKeyDown);
+                isKeyDownHandlerAttached = false;
+
+                activeElement = null;
+            }
+        }
+
+        // Add mouse event handlers
+        targetElement.addEventListener("mousemove", handleMouseMove);
+        targetElement.addEventListener("mouseleave", handleMouseLeave);
+
+        // Reset zoom when click on another tab
+        elements.img2imgTabs.addEventListener("click", resetZoom);
+        elements.img2imgTabs.addEventListener("click", () => {
+            // targetElement.style.width = "";
+            if (parseInt(targetElement.style.width) > 865) {
+                setTimeout(fitToElement, 0);
+            }
+        });
+
+        targetElement.addEventListener("wheel", e => {
+            // change zoom level
+            const operation = e.deltaY > 0 ? "-" : "+";
+            changeZoomLevel(operation, e);
+
+            // Handle brush size adjustment with ctrl key pressed
+            if (
+                (hotkeysConfig.canvas_swap_controls && e.shiftKey) ||
+                (!hotkeysConfig.canvas_swap_controls &&
+                    (e.ctrlKey || e.metaKey))
+            ) {
+                e.preventDefault();
+
+                // Increase or decrease brush size based on scroll direction
+                adjustBrushSize(elemId, e.deltaY);
+            }
+        });
+
+        // Handle the move event for pan functionality. Updates the panX and panY variables and applies the new transform to the target element.
+        function handleMoveKeyDown(e) {
+            if (e.code === hotkeysConfig.canvas_hotkey_move) {
+                if (!e.ctrlKey && !e.metaKey && isKeyDownHandlerAttached) {
+                    e.preventDefault();
+                    document.activeElement.blur();
+                    isMoving = true;
+                }
+            }
+        }
+
+        function handleMoveKeyUp(e) {
+            if (e.code === hotkeysConfig.canvas_hotkey_move) {
+                isMoving = false;
+            }
+        }
+
+        document.addEventListener("keydown", handleMoveKeyDown);
+        document.addEventListener("keyup", handleMoveKeyUp);
+
+        // Detect zoom level and update the pan speed.
+        function updatePanPosition(movementX, movementY) {
+            let panSpeed = 2;
+
+            if (elemData[elemId].zoomLevel > 8) {
+                panSpeed = 3.5;
+            }
+
+            elemData[elemId].panX += movementX * panSpeed;
+            elemData[elemId].panY += movementY * panSpeed;
+
+            // Delayed redraw of an element
+            requestAnimationFrame(() => {
+                targetElement.style.transform = `translate(${elemData[elemId].panX}px, ${elemData[elemId].panY}px) scale(${elemData[elemId].zoomLevel})`;
+                toggleOverlap("on");
+            });
+        }
+
+        function handleMoveByKey(e) {
+            if (isMoving && elemId === activeElement) {
+                updatePanPosition(e.movementX, e.movementY);
+                targetElement.style.pointerEvents = "none";
+            } else {
+                targetElement.style.pointerEvents = "auto";
+            }
+        }
+
+        // Prevents sticking to the mouse
+        window.onblur = function() {
+            isMoving = false;
+        };
+
+        gradioApp().addEventListener("mousemove", handleMoveByKey);
+    }
+
+    applyZoomAndPan(elementIDs.sketch);
+    applyZoomAndPan(elementIDs.inpaint);
+    applyZoomAndPan(elementIDs.inpaintSketch);
+
+    // Make the function global so that other extensions can take advantage of this solution
+    window.applyZoomAndPan = applyZoomAndPan;
+});

extensions-builtin/canvas-zoom-and-pan/scripts/hotkey_config.py

@@ -0,0 +1,10 @@
+from modules import shared
+
+shared.options_templates.update(shared.options_section(('canvas_hotkey', "Canvas Hotkeys"), {
+    "canvas_hotkey_move": shared.OptionInfo("F", "Moving the canvas"),
+    "canvas_hotkey_fullscreen": shared.OptionInfo("S", "Fullscreen Mode, maximizes the picture so that it fits into the screen and stretches it to its full width "),
+    "canvas_hotkey_reset": shared.OptionInfo("R", "Reset zoom and canvas positon"),
+    "canvas_hotkey_overlap": shared.OptionInfo("O", "Toggle overlap ( Technical button, neededs for testing )"),
+    "canvas_show_tooltip": shared.OptionInfo(True, "Enable tooltip on the canvas"),
+    "canvas_swap_controls": shared.OptionInfo(False, "Swap hotkey combinations for Zoom and Adjust brush resize"),
+}))

extensions-builtin/canvas-zoom-and-pan/style.css

@@ -0,0 +1,63 @@
+.tooltip-info {
+  position: absolute;
+  top: 10px;
+  left: 10px;
+  cursor: help;
+  background-color: rgba(0, 0, 0, 0.3);
+  width: 20px;
+  height: 20px;
+  border-radius: 50%;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  flex-direction: column; 
+
+  z-index: 100;
+}
+
+.tooltip-info::after {
+  content: '';
+  display: block;
+  width: 2px;
+  height: 7px;
+  background-color: white;
+  margin-top: 2px; 
+}
+
+.tooltip-info::before {
+  content: '';
+  display: block;
+  width: 2px;
+  height: 2px;
+  background-color: white;
+}
+
+.tooltip-content {
+  display: none;
+  background-color: #f9f9f9; 
+  color: #333; 
+  border: 1px solid #ddd;
+  padding: 15px; 
+  position: absolute;
+  top: 40px;
+  left: 10px;
+  width: 250px;
+  font-size: 16px;
+  opacity: 0;
+  border-radius: 8px;
+  box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); 
+
+  z-index: 100;
+}
+
+.tooltip:hover .tooltip-content {
+  display: block;
+  animation: fadeIn 0.5s; 
+  opacity: 1;
+}
+
+@keyframes fadeIn {
+  from {opacity: 0;}
+  to {opacity: 1;}
+}
+

extensions-builtin/extra-options-section/scripts/extra_options_section.py

@@ -0,0 +1,48 @@
+import gradio as gr
+from modules import scripts, shared, ui_components, ui_settings
+from modules.ui_components import FormColumn
+
+
+class ExtraOptionsSection(scripts.Script):
+    section = "extra_options"
+
+    def __init__(self):
+        self.comps = None
+        self.setting_names = None
+
+    def title(self):
+        return "Extra options"
+
+    def show(self, is_img2img):
+        return scripts.AlwaysVisible
+
+    def ui(self, is_img2img):
+        self.comps = []
+        self.setting_names = []
+
+        with gr.Blocks() as interface:
+            with gr.Accordion("Options", open=False) if shared.opts.extra_options_accordion and shared.opts.extra_options else gr.Group(), gr.Row():
+                for setting_name in shared.opts.extra_options:
+                    with FormColumn():
+                        comp = ui_settings.create_setting_component(setting_name)
+
+                    self.comps.append(comp)
+                    self.setting_names.append(setting_name)
+
+        def get_settings_values():
+            return [ui_settings.get_value_for_setting(key) for key in self.setting_names]
+
+        interface.load(fn=get_settings_values, inputs=[], outputs=self.comps, queue=False, show_progress=False)
+
+        return self.comps
+
+    def before_process(self, p, *args):
+        for name, value in zip(self.setting_names, args):
+            if name not in p.override_settings:
+                p.override_settings[name] = value
+
+
+shared.options_templates.update(shared.options_section(('ui', "User interface"), {
+    "extra_options": shared.OptionInfo([], "Options in main UI", ui_components.DropdownMulti, lambda: {"choices": list(shared.opts.data_labels.keys())}).js("info", "settingsHintsShowQuicksettings").info("setting entries that also appear in txt2img/img2img interfaces").needs_restart(),
+    "extra_options_accordion": shared.OptionInfo(False, "Place options in main UI into an accordion")
+}))

extensions-builtin/prompt-bracket-checker/javascript/prompt-bracket-checker.js

@@ -1,110 +1,42 @@
 // Stable Diffusion WebUI - Bracket checker
-// Version 1.0
-// By Hingashi no Florin/Bwin4L
+// By Hingashi no Florin/Bwin4L & @akx
 // Counts open and closed brackets (round, square, curly) in the prompt and negative prompt text boxes in the txt2img and img2img tabs.
 // If there's a mismatch, the keyword counter turns red and if you hover on it, a tooltip tells you what's wrong.
 
-function checkBrackets(evt, textArea, counterElt) {
-  errorStringParen  = '(...) - Different number of opening and closing parentheses detected.\n';
-  errorStringSquare = '[...] - Different number of opening and closing square brackets detected.\n';
-  errorStringCurly  = '{...} - Different number of opening and closing curly brackets detected.\n';
-
-  openBracketRegExp = /\(/g;
-  closeBracketRegExp = /\)/g;
-
-  openSquareBracketRegExp = /\[/g;
-  closeSquareBracketRegExp = /\]/g;
-
-  openCurlyBracketRegExp = /\{/g;
-  closeCurlyBracketRegExp = /\}/g;
-
-  totalOpenBracketMatches = 0;
-  totalCloseBracketMatches = 0;
-  totalOpenSquareBracketMatches = 0;
-  totalCloseSquareBracketMatches = 0;
-  totalOpenCurlyBracketMatches = 0;
-  totalCloseCurlyBracketMatches = 0;
-
-  openBracketMatches = textArea.value.match(openBracketRegExp);
-  if(openBracketMatches) {
-    totalOpenBracketMatches = openBracketMatches.length;
-  }
-
-  closeBracketMatches = textArea.value.match(closeBracketRegExp);
-  if(closeBracketMatches) {
-    totalCloseBracketMatches = closeBracketMatches.length;
-  }
-
-  openSquareBracketMatches = textArea.value.match(openSquareBracketRegExp);
-  if(openSquareBracketMatches) {
-    totalOpenSquareBracketMatches = openSquareBracketMatches.length;
-  }
-
-  closeSquareBracketMatches = textArea.value.match(closeSquareBracketRegExp);
-  if(closeSquareBracketMatches) {
-    totalCloseSquareBracketMatches = closeSquareBracketMatches.length;
-  }
-
-  openCurlyBracketMatches = textArea.value.match(openCurlyBracketRegExp);
-  if(openCurlyBracketMatches) {
-    totalOpenCurlyBracketMatches = openCurlyBracketMatches.length;
-  }
-
-  closeCurlyBracketMatches = textArea.value.match(closeCurlyBracketRegExp);
-  if(closeCurlyBracketMatches) {
-    totalCloseCurlyBracketMatches = closeCurlyBracketMatches.length;
-  }
-
-  if(totalOpenBracketMatches != totalCloseBracketMatches) {
-    if(!counterElt.title.includes(errorStringParen)) {
-      counterElt.title += errorStringParen;
-    }
-  } else {
-    counterElt.title = counterElt.title.replace(errorStringParen, '');
-  }
-
-  if(totalOpenSquareBracketMatches != totalCloseSquareBracketMatches) {
-    if(!counterElt.title.includes(errorStringSquare)) {
-      counterElt.title += errorStringSquare;
-    }
-  } else {
-    counterElt.title = counterElt.title.replace(errorStringSquare, '');
-  }
-
-  if(totalOpenCurlyBracketMatches != totalCloseCurlyBracketMatches) {
-    if(!counterElt.title.includes(errorStringCurly)) {
-      counterElt.title += errorStringCurly;
-    }
-  } else {
-    counterElt.title = counterElt.title.replace(errorStringCurly, '');
-  }
-
-  if(counterElt.title != '') {
-    counterElt.classList.add('error');
-  } else {
-    counterElt.classList.remove('error');
-  }
-}
-
-function setupBracketChecking(id_prompt, id_counter){
-    var textarea = gradioApp().querySelector("#" + id_prompt + " > label > textarea");
-    var counter = gradioApp().getElementById(id_counter)
-    textarea.addEventListener("input", function(evt){
-        checkBrackets(evt, textarea, counter)
+function checkBrackets(textArea, counterElt) {
+    var counts = {};
+    (textArea.value.match(/[(){}[\]]/g) || []).forEach(bracket => {
+        counts[bracket] = (counts[bracket] || 0) + 1;
     });
+    var errors = [];
+
+    function checkPair(open, close, kind) {
+        if (counts[open] !== counts[close]) {
+            errors.push(
+                `${open}...${close} - Detected ${counts[open] || 0} opening and ${counts[close] || 0} closing ${kind}.`
+            );
+        }
+    }
+
+    checkPair('(', ')', 'round brackets');
+    checkPair('[', ']', 'square brackets');
+    checkPair('{', '}', 'curly brackets');
+    counterElt.title = errors.join('\n');
+    counterElt.classList.toggle('error', errors.length !== 0);
 }
 
-var shadowRootLoaded = setInterval(function() {
-    var shadowRoot = document.querySelector('gradio-app').shadowRoot;
-    if(! shadowRoot)  return false;
+function setupBracketChecking(id_prompt, id_counter) {
+    var textarea = gradioApp().querySelector("#" + id_prompt + " > label > textarea");
+    var counter = gradioApp().getElementById(id_counter);
 
-    var shadowTextArea = shadowRoot.querySelectorAll('#txt2img_prompt > label > textarea');
-    if(shadowTextArea.length < 1)  return false;
+    if (textarea && counter) {
+        textarea.addEventListener("input", () => checkBrackets(textarea, counter));
+    }
+}
 
-    clearInterval(shadowRootLoaded);
-
-    setupBracketChecking('txt2img_prompt', 'txt2img_token_counter')
-    setupBracketChecking('txt2img_neg_prompt', 'txt2img_negative_token_counter')
-    setupBracketChecking('img2img_prompt', 'imgimg_token_counter')
-    setupBracketChecking('img2img_neg_prompt', 'img2img_negative_token_counter')
-}, 1000);
+onUiLoaded(function() {
+    setupBracketChecking('txt2img_prompt', 'txt2img_token_counter');
+    setupBracketChecking('txt2img_neg_prompt', 'txt2img_negative_token_counter');
+    setupBracketChecking('img2img_prompt', 'img2img_token_counter');
+    setupBracketChecking('img2img_neg_prompt', 'img2img_negative_token_counter');
+});

html/extra-networks-card.html

@@ -1,11 +1,14 @@
-<div class='card' {preview_html} onclick={card_clicked}>
+<div class='card' style={style} onclick={card_clicked} {sort_keys}>
+	{background_image}
+	{metadata_button}
 	<div class='actions'>
 		<div class='additional'>
 			<ul>
 				<a href="#" title="replace preview image with currently selected in gallery" onclick={save_card_preview}>replace preview</a>
 			</ul>
+			<span style="display:none" class='search_term{search_only}'>{search_term}</span>
 		</div>
 		<span class='name'>{name}</span>
+		<span class='description'>{description}</span>
 	</div>
 </div>
-

html/footer.html

@@ -1,10 +1,12 @@
 <div>
-        <a href="/docs">API</a>
+        <a href="{api_docs}">API</a>
          • 
         <a href="https://github.com/AUTOMATIC1111/stable-diffusion-webui">Github</a>
          • 
         <a href="https://gradio.app">Gradio</a>
          • 
+        <a href="#" onclick="showProfile('./internal/profile-startup'); return false;">Startup profile</a>
+         • 
         <a href="/" onclick="javascript:gradioApp().getElementById('settings_restart_gradio').click(); return false">Reload UI</a>
 </div>
 <br />

html/licenses.html

@@ -417,3 +417,274 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 </pre>
 
+<h2><a href="https://github.com/huggingface/diffusers/blob/c7da8fd23359a22d0df2741688b5b4f33c26df21/LICENSE">Scaled Dot Product Attention</a></h2>
+<small>Some small amounts of code borrowed and reworked.</small>
+<pre>
+   Copyright 2023 The HuggingFace Team. All rights reserved.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+</pre>
+
+<h2><a href="https://github.com/explosion/curated-transformers/blob/main/LICENSE">Curated transformers</a></h2>
+<small>The MPS workaround for nn.Linear on macOS 13.2.X is based on the MPS workaround for nn.Linear created by danieldk for Curated transformers</small>
+<pre>
+The MIT License (MIT)
+
+Copyright (C) 2021 ExplosionAI GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+</pre>
+
+<h2><a href="https://github.com/madebyollin/taesd/blob/main/LICENSE">TAESD</a></h2>
+<small>Tiny AutoEncoder for Stable Diffusion option for live previews</small>
+<pre>
+MIT License
+
+Copyright (c) 2023 Ollin Boer Bohan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+</pre>

javascript/aspectRatioOverlay.js

@@ -1,113 +1,113 @@
-
-let currentWidth = null;
-let currentHeight = null;
-let arFrameTimeout = setTimeout(function(){},0);
-
-function dimensionChange(e, is_width, is_height){
-
-	if(is_width){
-		currentWidth = e.target.value*1.0
-	}
-	if(is_height){
-		currentHeight = e.target.value*1.0
-	}
-
-	var inImg2img   = Boolean(gradioApp().querySelector("button.rounded-t-lg.border-gray-200"))
-
-	if(!inImg2img){
-		return;
-	}
-
-	var targetElement = null;
-
-    var tabIndex = get_tab_index('mode_img2img')
-	if(tabIndex == 0){ // img2img
-		targetElement = gradioApp().querySelector('div[data-testid=image] img');
-	} else if(tabIndex == 1){ //Sketch
-		targetElement = gradioApp().querySelector('#img2img_sketch div[data-testid=image] img');
-	} else if(tabIndex == 2){ // Inpaint
-		targetElement = gradioApp().querySelector('#img2maskimg div[data-testid=image] img');
-	} else if(tabIndex == 3){ // Inpaint sketch
-		targetElement = gradioApp().querySelector('#inpaint_sketch div[data-testid=image] img');
-	}
-	
-
-	if(targetElement){
-
-		var arPreviewRect = gradioApp().querySelector('#imageARPreview');
-		if(!arPreviewRect){
-		    arPreviewRect = document.createElement('div')
-		    arPreviewRect.id = "imageARPreview";
-		    gradioApp().getRootNode().appendChild(arPreviewRect)
-		}
-
-
-
-		var viewportOffset = targetElement.getBoundingClientRect();
-
-		viewportscale = Math.min(  targetElement.clientWidth/targetElement.naturalWidth, targetElement.clientHeight/targetElement.naturalHeight )
-
-		scaledx = targetElement.naturalWidth*viewportscale
-		scaledy = targetElement.naturalHeight*viewportscale
-
-		cleintRectTop    = (viewportOffset.top+window.scrollY)
-		cleintRectLeft   = (viewportOffset.left+window.scrollX)
-		cleintRectCentreY = cleintRectTop  + (targetElement.clientHeight/2)
-		cleintRectCentreX = cleintRectLeft + (targetElement.clientWidth/2)
-
-		viewRectTop    = cleintRectCentreY-(scaledy/2)
-		viewRectLeft   = cleintRectCentreX-(scaledx/2)
-		arRectWidth  = scaledx
-		arRectHeight = scaledy
-
-		arscale = Math.min(  arRectWidth/currentWidth, arRectHeight/currentHeight )
-		arscaledx = currentWidth*arscale
-		arscaledy = currentHeight*arscale
-
-		arRectTop    = cleintRectCentreY-(arscaledy/2)
-		arRectLeft   = cleintRectCentreX-(arscaledx/2)
-		arRectWidth  = arscaledx
-		arRectHeight = arscaledy
-
-	    arPreviewRect.style.top  = arRectTop+'px';
-	    arPreviewRect.style.left = arRectLeft+'px';
-	    arPreviewRect.style.width = arRectWidth+'px';
-	    arPreviewRect.style.height = arRectHeight+'px';
-
-	    clearTimeout(arFrameTimeout);
-	    arFrameTimeout = setTimeout(function(){
-	    	arPreviewRect.style.display = 'none';
-	    },2000);
-
-	    arPreviewRect.style.display = 'block';
-
-	}
-
-}
-
-
-onUiUpdate(function(){
-	var arPreviewRect = gradioApp().querySelector('#imageARPreview');
-	if(arPreviewRect){
-		arPreviewRect.style.display = 'none';
-	}
-	var inImg2img   = Boolean(gradioApp().querySelector("button.rounded-t-lg.border-gray-200"))
-	if(inImg2img){
-		let inputs = gradioApp().querySelectorAll('input');
-		inputs.forEach(function(e){
-		    var is_width = e.parentElement.id == "img2img_width"
-		    var is_height = e.parentElement.id == "img2img_height"
-
-			if((is_width || is_height) && !e.classList.contains('scrollwatch')){
-				e.addEventListener('input', function(e){dimensionChange(e, is_width, is_height)} )
-				e.classList.add('scrollwatch')
-			}
-			if(is_width){
-				currentWidth = e.value*1.0
-			}
-			if(is_height){
-				currentHeight = e.value*1.0
-			}
-		})
-	}
-});
+
+let currentWidth = null;
+let currentHeight = null;
+let arFrameTimeout = setTimeout(function() {}, 0);
+
+function dimensionChange(e, is_width, is_height) {
+
+    if (is_width) {
+        currentWidth = e.target.value * 1.0;
+    }
+    if (is_height) {
+        currentHeight = e.target.value * 1.0;
+    }
+
+    var inImg2img = gradioApp().querySelector("#tab_img2img").style.display == "block";
+
+    if (!inImg2img) {
+        return;
+    }
+
+    var targetElement = null;
+
+    var tabIndex = get_tab_index('mode_img2img');
+    if (tabIndex == 0) { // img2img
+        targetElement = gradioApp().querySelector('#img2img_image div[data-testid=image] img');
+    } else if (tabIndex == 1) { //Sketch
+        targetElement = gradioApp().querySelector('#img2img_sketch div[data-testid=image] img');
+    } else if (tabIndex == 2) { // Inpaint
+        targetElement = gradioApp().querySelector('#img2maskimg div[data-testid=image] img');
+    } else if (tabIndex == 3) { // Inpaint sketch
+        targetElement = gradioApp().querySelector('#inpaint_sketch div[data-testid=image] img');
+    }
+
+
+    if (targetElement) {
+
+        var arPreviewRect = gradioApp().querySelector('#imageARPreview');
+        if (!arPreviewRect) {
+            arPreviewRect = document.createElement('div');
+            arPreviewRect.id = "imageARPreview";
+            gradioApp().appendChild(arPreviewRect);
+        }
+
+
+
+        var viewportOffset = targetElement.getBoundingClientRect();
+
+        var viewportscale = Math.min(targetElement.clientWidth / targetElement.naturalWidth, targetElement.clientHeight / targetElement.naturalHeight);
+
+        var scaledx = targetElement.naturalWidth * viewportscale;
+        var scaledy = targetElement.naturalHeight * viewportscale;
+
+        var cleintRectTop = (viewportOffset.top + window.scrollY);
+        var cleintRectLeft = (viewportOffset.left + window.scrollX);
+        var cleintRectCentreY = cleintRectTop + (targetElement.clientHeight / 2);
+        var cleintRectCentreX = cleintRectLeft + (targetElement.clientWidth / 2);
+
+        var arscale = Math.min(scaledx / currentWidth, scaledy / currentHeight);
+        var arscaledx = currentWidth * arscale;
+        var arscaledy = currentHeight * arscale;
+
+        var arRectTop = cleintRectCentreY - (arscaledy / 2);
+        var arRectLeft = cleintRectCentreX - (arscaledx / 2);
+        var arRectWidth = arscaledx;
+        var arRectHeight = arscaledy;
+
+        arPreviewRect.style.top = arRectTop + 'px';
+        arPreviewRect.style.left = arRectLeft + 'px';
+        arPreviewRect.style.width = arRectWidth + 'px';
+        arPreviewRect.style.height = arRectHeight + 'px';
+
+        clearTimeout(arFrameTimeout);
+        arFrameTimeout = setTimeout(function() {
+            arPreviewRect.style.display = 'none';
+        }, 2000);
+
+        arPreviewRect.style.display = 'block';
+
+    }
+
+}
+
+
+onAfterUiUpdate(function() {
+    var arPreviewRect = gradioApp().querySelector('#imageARPreview');
+    if (arPreviewRect) {
+        arPreviewRect.style.display = 'none';
+    }
+    var tabImg2img = gradioApp().querySelector("#tab_img2img");
+    if (tabImg2img) {
+        var inImg2img = tabImg2img.style.display == "block";
+        if (inImg2img) {
+            let inputs = gradioApp().querySelectorAll('input');
+            inputs.forEach(function(e) {
+                var is_width = e.parentElement.id == "img2img_width";
+                var is_height = e.parentElement.id == "img2img_height";
+
+                if ((is_width || is_height) && !e.classList.contains('scrollwatch')) {
+                    e.addEventListener('input', function(e) {
+                        dimensionChange(e, is_width, is_height);
+                    });
+                    e.classList.add('scrollwatch');
+                }
+                if (is_width) {
+                    currentWidth = e.value * 1.0;
+                }
+                if (is_height) {
+                    currentHeight = e.value * 1.0;
+                }
+            });
+        }
+    }
+});

javascript/contextMenus.js

@@ -1,177 +1,176 @@
-
-contextMenuInit = function(){
-  let eventListenerApplied=false;
-  let menuSpecs = new Map();
-
-  const uid = function(){
-    return Date.now().toString(36) + Math.random().toString(36).substr(2);
-  }
-
-  function showContextMenu(event,element,menuEntries){
-    let posx = event.clientX + document.body.scrollLeft + document.documentElement.scrollLeft;
-    let posy = event.clientY + document.body.scrollTop + document.documentElement.scrollTop;
-
-    let oldMenu = gradioApp().querySelector('#context-menu')
-    if(oldMenu){
-      oldMenu.remove()
-    }
-
-    let tabButton = uiCurrentTab
-    let baseStyle = window.getComputedStyle(tabButton)
-
-    const contextMenu = document.createElement('nav')
-    contextMenu.id = "context-menu"
-    contextMenu.style.background = baseStyle.background
-    contextMenu.style.color = baseStyle.color
-    contextMenu.style.fontFamily = baseStyle.fontFamily
-    contextMenu.style.top = posy+'px'
-    contextMenu.style.left = posx+'px'
-
-
-
-    const contextMenuList = document.createElement('ul')
-    contextMenuList.className = 'context-menu-items';
-    contextMenu.append(contextMenuList);
-
-    menuEntries.forEach(function(entry){
-      let contextMenuEntry = document.createElement('a')
-      contextMenuEntry.innerHTML = entry['name']
-      contextMenuEntry.addEventListener("click", function(e) {
-        entry['func']();
-      })
-      contextMenuList.append(contextMenuEntry);
-
-    })
-
-    gradioApp().getRootNode().appendChild(contextMenu)
-
-    let menuWidth = contextMenu.offsetWidth + 4;
-    let menuHeight = contextMenu.offsetHeight + 4;
-
-    let windowWidth = window.innerWidth;
-    let windowHeight = window.innerHeight;
-
-    if ( (windowWidth - posx) < menuWidth ) {
-      contextMenu.style.left = windowWidth - menuWidth + "px";
-    }
-
-    if ( (windowHeight - posy) < menuHeight ) {
-      contextMenu.style.top = windowHeight - menuHeight + "px";
-    }
-
-  }
-
-  function appendContextMenuOption(targetElementSelector,entryName,entryFunction){
-
-    currentItems = menuSpecs.get(targetElementSelector)
-
-    if(!currentItems){
-      currentItems = []
-      menuSpecs.set(targetElementSelector,currentItems);
-    }
-    let newItem = {'id':targetElementSelector+'_'+uid(),
-                   'name':entryName,
-                   'func':entryFunction,
-                   'isNew':true}
-
-    currentItems.push(newItem)
-    return newItem['id']
-  }
-
-  function removeContextMenuOption(uid){
-    menuSpecs.forEach(function(v,k) {
-      let index = -1
-      v.forEach(function(e,ei){if(e['id']==uid){index=ei}})
-      if(index>=0){
-        v.splice(index, 1);
-      }
-    })
-  }
-
-  function addContextMenuEventListener(){
-    if(eventListenerApplied){
-      return;
-    }
-    gradioApp().addEventListener("click", function(e) {
-      let source = e.composedPath()[0]
-      if(source.id && source.id.indexOf('check_progress')>-1){
-        return
-      }
-
-      let oldMenu = gradioApp().querySelector('#context-menu')
-      if(oldMenu){
-        oldMenu.remove()
-      }
-    });
-    gradioApp().addEventListener("contextmenu", function(e) {
-      let oldMenu = gradioApp().querySelector('#context-menu')
-      if(oldMenu){
-        oldMenu.remove()
-      }
-      menuSpecs.forEach(function(v,k) {
-        if(e.composedPath()[0].matches(k)){
-          showContextMenu(e,e.composedPath()[0],v)
-          e.preventDefault()
-          return
-        }
-      })
-    });
-    eventListenerApplied=true
-
-  }
-
-  return [appendContextMenuOption, removeContextMenuOption, addContextMenuEventListener]
-}
-
-initResponse = contextMenuInit();
-appendContextMenuOption     = initResponse[0];
-removeContextMenuOption     = initResponse[1];
-addContextMenuEventListener = initResponse[2];
-
-(function(){
-  //Start example Context Menu Items
-  let generateOnRepeat = function(genbuttonid,interruptbuttonid){
-    let genbutton = gradioApp().querySelector(genbuttonid);
-    let interruptbutton = gradioApp().querySelector(interruptbuttonid);
-    if(!interruptbutton.offsetParent){
-      genbutton.click();
-    }
-    clearInterval(window.generateOnRepeatInterval)
-    window.generateOnRepeatInterval = setInterval(function(){
-      if(!interruptbutton.offsetParent){
-        genbutton.click();
-      }
-    },
-    500)
-  }
-
-  appendContextMenuOption('#txt2img_generate','Generate forever',function(){
-    generateOnRepeat('#txt2img_generate','#txt2img_interrupt');
-  })
-  appendContextMenuOption('#img2img_generate','Generate forever',function(){
-    generateOnRepeat('#img2img_generate','#img2img_interrupt');
-  })
-
-  let cancelGenerateForever = function(){
-    clearInterval(window.generateOnRepeatInterval)
-  }
-
-  appendContextMenuOption('#txt2img_interrupt','Cancel generate forever',cancelGenerateForever)
-  appendContextMenuOption('#txt2img_generate', 'Cancel generate forever',cancelGenerateForever)
-  appendContextMenuOption('#img2img_interrupt','Cancel generate forever',cancelGenerateForever)
-  appendContextMenuOption('#img2img_generate', 'Cancel generate forever',cancelGenerateForever)
-
-  appendContextMenuOption('#roll','Roll three',
-    function(){
-      let rollbutton = get_uiCurrentTabContent().querySelector('#roll');
-      setTimeout(function(){rollbutton.click()},100)
-      setTimeout(function(){rollbutton.click()},200)
-      setTimeout(function(){rollbutton.click()},300)
-    }
-  )
-})();
-//End example Context Menu Items
-
-onUiUpdate(function(){
-  addContextMenuEventListener()
-});
+
+var contextMenuInit = function() {
+    let eventListenerApplied = false;
+    let menuSpecs = new Map();
+
+    const uid = function() {
+        return Date.now().toString(36) + Math.random().toString(36).substring(2);
+    };
+
+    function showContextMenu(event, element, menuEntries) {
+        let posx = event.clientX + document.body.scrollLeft + document.documentElement.scrollLeft;
+        let posy = event.clientY + document.body.scrollTop + document.documentElement.scrollTop;
+
+        let oldMenu = gradioApp().querySelector('#context-menu');
+        if (oldMenu) {
+            oldMenu.remove();
+        }
+
+        let baseStyle = window.getComputedStyle(uiCurrentTab);
+
+        const contextMenu = document.createElement('nav');
+        contextMenu.id = "context-menu";
+        contextMenu.style.background = baseStyle.background;
+        contextMenu.style.color = baseStyle.color;
+        contextMenu.style.fontFamily = baseStyle.fontFamily;
+        contextMenu.style.top = posy + 'px';
+        contextMenu.style.left = posx + 'px';
+
+
+
+        const contextMenuList = document.createElement('ul');
+        contextMenuList.className = 'context-menu-items';
+        contextMenu.append(contextMenuList);
+
+        menuEntries.forEach(function(entry) {
+            let contextMenuEntry = document.createElement('a');
+            contextMenuEntry.innerHTML = entry['name'];
+            contextMenuEntry.addEventListener("click", function() {
+                entry['func']();
+            });
+            contextMenuList.append(contextMenuEntry);
+
+        });
+
+        gradioApp().appendChild(contextMenu);
+
+        let menuWidth = contextMenu.offsetWidth + 4;
+        let menuHeight = contextMenu.offsetHeight + 4;
+
+        let windowWidth = window.innerWidth;
+        let windowHeight = window.innerHeight;
+
+        if ((windowWidth - posx) < menuWidth) {
+            contextMenu.style.left = windowWidth - menuWidth + "px";
+        }
+
+        if ((windowHeight - posy) < menuHeight) {
+            contextMenu.style.top = windowHeight - menuHeight + "px";
+        }
+
+    }
+
+    function appendContextMenuOption(targetElementSelector, entryName, entryFunction) {
+
+        var currentItems = menuSpecs.get(targetElementSelector);
+
+        if (!currentItems) {
+            currentItems = [];
+            menuSpecs.set(targetElementSelector, currentItems);
+        }
+        let newItem = {
+            id: targetElementSelector + '_' + uid(),
+            name: entryName,
+            func: entryFunction,
+            isNew: true
+        };
+
+        currentItems.push(newItem);
+        return newItem['id'];
+    }
+
+    function removeContextMenuOption(uid) {
+        menuSpecs.forEach(function(v) {
+            let index = -1;
+            v.forEach(function(e, ei) {
+                if (e['id'] == uid) {
+                    index = ei;
+                }
+            });
+            if (index >= 0) {
+                v.splice(index, 1);
+            }
+        });
+    }
+
+    function addContextMenuEventListener() {
+        if (eventListenerApplied) {
+            return;
+        }
+        gradioApp().addEventListener("click", function(e) {
+            if (!e.isTrusted) {
+                return;
+            }
+
+            let oldMenu = gradioApp().querySelector('#context-menu');
+            if (oldMenu) {
+                oldMenu.remove();
+            }
+        });
+        gradioApp().addEventListener("contextmenu", function(e) {
+            let oldMenu = gradioApp().querySelector('#context-menu');
+            if (oldMenu) {
+                oldMenu.remove();
+            }
+            menuSpecs.forEach(function(v, k) {
+                if (e.composedPath()[0].matches(k)) {
+                    showContextMenu(e, e.composedPath()[0], v);
+                    e.preventDefault();
+                }
+            });
+        });
+        eventListenerApplied = true;
+
+    }
+
+    return [appendContextMenuOption, removeContextMenuOption, addContextMenuEventListener];
+};
+
+var initResponse = contextMenuInit();
+var appendContextMenuOption = initResponse[0];
+var removeContextMenuOption = initResponse[1];
+var addContextMenuEventListener = initResponse[2];
+
+(function() {
+    //Start example Context Menu Items
+    let generateOnRepeat = function(genbuttonid, interruptbuttonid) {
+        let genbutton = gradioApp().querySelector(genbuttonid);
+        let interruptbutton = gradioApp().querySelector(interruptbuttonid);
+        if (!interruptbutton.offsetParent) {
+            genbutton.click();
+        }
+        clearInterval(window.generateOnRepeatInterval);
+        window.generateOnRepeatInterval = setInterval(function() {
+            if (!interruptbutton.offsetParent) {
+                genbutton.click();
+            }
+        },
+        500);
+    };
+
+    let generateOnRepeat_txt2img = function() {
+        generateOnRepeat('#txt2img_generate', '#txt2img_interrupt');
+    };
+
+    let generateOnRepeat_img2img = function() {
+        generateOnRepeat('#img2img_generate', '#img2img_interrupt');
+    };
+
+    appendContextMenuOption('#txt2img_generate', 'Generate forever', generateOnRepeat_txt2img);
+    appendContextMenuOption('#txt2img_interrupt', 'Generate forever', generateOnRepeat_txt2img);
+    appendContextMenuOption('#img2img_generate', 'Generate forever', generateOnRepeat_img2img);
+    appendContextMenuOption('#img2img_interrupt', 'Generate forever', generateOnRepeat_img2img);
+
+    let cancelGenerateForever = function() {
+        clearInterval(window.generateOnRepeatInterval);
+    };
+
+    appendContextMenuOption('#txt2img_interrupt', 'Cancel generate forever', cancelGenerateForever);
+    appendContextMenuOption('#txt2img_generate', 'Cancel generate forever', cancelGenerateForever);
+    appendContextMenuOption('#img2img_interrupt', 'Cancel generate forever', cancelGenerateForever);
+    appendContextMenuOption('#img2img_generate', 'Cancel generate forever', cancelGenerateForever);
+
+})();
+//End example Context Menu Items
+
+onAfterUiUpdate(addContextMenuEventListener);

javascript/dragdrop.js

@@ -1,11 +1,11 @@
 // allows drag-dropping files into gradio image elements, and also pasting images from clipboard
 
-function isValidImageList( files ) {
+function isValidImageList(files) {
     return files && files?.length === 1 && ['image/png', 'image/gif', 'image/jpeg'].includes(files[0].type);
 }
 
-function dropReplaceImage( imgWrap, files ) {
-    if ( ! isValidImageList( files ) ) {
+function dropReplaceImage(imgWrap, files) {
+    if (!isValidImageList(files)) {
         return;
     }
 
@@ -14,46 +14,61 @@ function dropReplaceImage( imgWrap, files ) {
     imgWrap.querySelector('.modify-upload button + button, .touch-none + div button + button')?.click();
     const callback = () => {
         const fileInput = imgWrap.querySelector('input[type="file"]');
-        if ( fileInput ) {
-            if ( files.length === 0 ) {
+        if (fileInput) {
+            if (files.length === 0) {
                 files = new DataTransfer();
                 files.items.add(tmpFile);
                 fileInput.files = files.files;
             } else {
                 fileInput.files = files;
             }
-            fileInput.dispatchEvent(new Event('change'));   
+            fileInput.dispatchEvent(new Event('change'));
         }
     };
-    
-    if ( imgWrap.closest('#pnginfo_image') ) {
+
+    if (imgWrap.closest('#pnginfo_image')) {
         // special treatment for PNG Info tab, wait for fetch request to finish
         const oldFetch = window.fetch;
-        window.fetch = async (input, options) => {
+        window.fetch = async(input, options) => {
             const response = await oldFetch(input, options);
-            if ( 'api/predict/' === input ) {
+            if ('api/predict/' === input) {
                 const content = await response.text();
                 window.fetch = oldFetch;
-                window.requestAnimationFrame( () => callback() );
+                window.requestAnimationFrame(() => callback());
                 return new Response(content, {
                     status: response.status,
                     statusText: response.statusText,
                     headers: response.headers
-                })
+                });
             }
             return response;
-        };        
+        };
     } else {
-        window.requestAnimationFrame( () => callback() );
+        window.requestAnimationFrame(() => callback());
     }
 }
 
+function eventHasFiles(e) {
+    if (!e.dataTransfer || !e.dataTransfer.files) return false;
+    if (e.dataTransfer.files.length > 0) return true;
+    if (e.dataTransfer.items.length > 0 && e.dataTransfer.items[0].kind == "file") return true;
+
+    return false;
+}
+
+function dragDropTargetIsPrompt(target) {
+    if (target?.placeholder && target?.placeholder.indexOf("Prompt") >= 0) return true;
+    if (target?.parentNode?.parentNode?.className?.indexOf("prompt") > 0) return true;
+    return false;
+}
+
 window.document.addEventListener('dragover', e => {
     const target = e.composedPath()[0];
-    const imgWrap = target.closest('[data-testid="image"]');
-    if ( !imgWrap && target.placeholder && target.placeholder.indexOf("Prompt") == -1) {
-        return;
-    }
+    if (!eventHasFiles(e)) return;
+
+    var targetImage = target.closest('[data-testid="image"]');
+    if (!dragDropTargetIsPrompt(target) && !targetImage) return;
+
     e.stopPropagation();
     e.preventDefault();
     e.dataTransfer.dropEffect = 'copy';
@@ -61,37 +76,55 @@ window.document.addEventListener('dragover', e => {
 
 window.document.addEventListener('drop', e => {
     const target = e.composedPath()[0];
-    if (target.placeholder.indexOf("Prompt") == -1) {
+    if (!eventHasFiles(e)) return;
+
+    if (dragDropTargetIsPrompt(target)) {
+        e.stopPropagation();
+        e.preventDefault();
+
+        let prompt_target = get_tab_index('tabs') == 1 ? "img2img_prompt_image" : "txt2img_prompt_image";
+
+        const imgParent = gradioApp().getElementById(prompt_target);
+        const files = e.dataTransfer.files;
+        const fileInput = imgParent.querySelector('input[type="file"]');
+        if (fileInput) {
+            fileInput.files = files;
+            fileInput.dispatchEvent(new Event('change'));
+        }
+    }
+
+    var targetImage = target.closest('[data-testid="image"]');
+    if (targetImage) {
+        e.stopPropagation();
+        e.preventDefault();
+        const files = e.dataTransfer.files;
+        dropReplaceImage(targetImage, files);
         return;
     }
-    const imgWrap = target.closest('[data-testid="image"]');
-    if ( !imgWrap ) {
-        return;
-    }
-    e.stopPropagation();
-    e.preventDefault();
-    const files = e.dataTransfer.files;
-    dropReplaceImage( imgWrap, files );
 });
 
 window.addEventListener('paste', e => {
     const files = e.clipboardData.files;
-    if ( ! isValidImageList( files ) ) {
+    if (!isValidImageList(files)) {
         return;
     }
 
     const visibleImageFields = [...gradioApp().querySelectorAll('[data-testid="image"]')]
-        .filter(el => uiElementIsVisible(el));
-    if ( ! visibleImageFields.length ) {
+        .filter(el => uiElementIsVisible(el))
+        .sort((a, b) => uiElementInSight(b) - uiElementInSight(a));
+
+
+    if (!visibleImageFields.length) {
         return;
     }
-    
+
     const firstFreeImageField = visibleImageFields
         .filter(el => el.querySelector('input[type=file]'))?.[0];
 
     dropReplaceImage(
         firstFreeImageField ?
-        firstFreeImageField :
-        visibleImageFields[visibleImageFields.length - 1]
-    , files );
+            firstFreeImageField :
+            visibleImageFields[visibleImageFields.length - 1]
+        , files
+    );
 });

javascript/edit-attention.js

@@ -1,96 +1,120 @@
-function keyupEditAttention(event){
-	let target = event.originalTarget || event.composedPath()[0];
-	if (!target.matches("[id*='_toprow'] textarea.gr-text-input[placeholder]")) return;
-	if (! (event.metaKey || event.ctrlKey)) return;
-
-	let isPlus = event.key == "ArrowUp"
-	let isMinus = event.key == "ArrowDown"
-	if (!isPlus && !isMinus) return;
-
-	let selectionStart = target.selectionStart;
-	let selectionEnd = target.selectionEnd;
-	let text = target.value;
-
-    function selectCurrentParenthesisBlock(OPEN, CLOSE){
-        if (selectionStart !== selectionEnd) return false;
-
-		// Find opening parenthesis around current cursor
-		const before = text.substring(0, selectionStart);
-		let beforeParen = before.lastIndexOf(OPEN);
-		if (beforeParen == -1) return  false;
-		let beforeParenClose = before.lastIndexOf(CLOSE);
-		while (beforeParenClose !== -1 && beforeParenClose > beforeParen) {
-			beforeParen = before.lastIndexOf(OPEN, beforeParen - 1);
-			beforeParenClose = before.lastIndexOf(CLOSE, beforeParenClose - 1);
-		}
-
-		// Find closing parenthesis around current cursor
-		const after = text.substring(selectionStart);
-		let afterParen = after.indexOf(CLOSE);
-		if (afterParen == -1) return  false;
-		let afterParenOpen = after.indexOf(OPEN);
-		while (afterParenOpen !== -1 && afterParen > afterParenOpen) {
-			afterParen = after.indexOf(CLOSE, afterParen + 1);
-			afterParenOpen = after.indexOf(OPEN, afterParenOpen + 1);
-		}
-		if (beforeParen === -1 || afterParen === -1) return false;
-
-		// Set the selection to the text between the parenthesis
-		const parenContent = text.substring(beforeParen + 1, selectionStart + afterParen);
-		const lastColon = parenContent.lastIndexOf(":");
-		selectionStart = beforeParen + 1;
-		selectionEnd = selectionStart + lastColon;
-		target.setSelectionRange(selectionStart, selectionEnd);
-		return true;
-    }
-
-	// If the user hasn't selected anything, let's select their current parenthesis block
-    if(! selectCurrentParenthesisBlock('<', '>')){
-        selectCurrentParenthesisBlock('(', ')')
-    }
-
-	event.preventDefault();
-
-    closeCharacter = ')'
-    delta = opts.keyedit_precision_attention
-
-    if (selectionStart > 0 && text[selectionStart - 1] == '<'){
-        closeCharacter = '>'
-        delta = opts.keyedit_precision_extra
-    } else if (selectionStart == 0 || text[selectionStart - 1] != "(") {
-
-        // do not include spaces at the end
-        while(selectionEnd > selectionStart && text[selectionEnd-1] == ' '){
-            selectionEnd -= 1;
-        }
-        if(selectionStart == selectionEnd){
-            return
-        }
-
-        text = text.slice(0, selectionStart) + "(" + text.slice(selectionStart, selectionEnd) + ":1.0)" + text.slice(selectionEnd);
-
-        selectionStart += 1;
-        selectionEnd += 1;
-    }
-
-	end = text.slice(selectionEnd + 1).indexOf(closeCharacter) + 1;
-	weight = parseFloat(text.slice(selectionEnd + 1, selectionEnd + 1 + end));
-	if (isNaN(weight)) return;
-
-	weight += isPlus ? delta : -delta;
-	weight = parseFloat(weight.toPrecision(12));
-	if(String(weight).length == 1) weight += ".0"
-
-	text = text.slice(0, selectionEnd + 1) + weight + text.slice(selectionEnd + 1 + end - 1);
-
-	target.focus();
-	target.value = text;
-	target.selectionStart = selectionStart;
-	target.selectionEnd = selectionEnd;
-
-	updateInput(target)
-}
-
-addEventListener('keydown', (event) => {
-    keyupEditAttention(event);
-});
+function keyupEditAttention(event) {
+    let target = event.originalTarget || event.composedPath()[0];
+    if (!target.matches("*:is([id*='_toprow'] [id*='_prompt'], .prompt) textarea")) return;
+    if (!(event.metaKey || event.ctrlKey)) return;
+
+    let isPlus = event.key == "ArrowUp";
+    let isMinus = event.key == "ArrowDown";
+    if (!isPlus && !isMinus) return;
+
+    let selectionStart = target.selectionStart;
+    let selectionEnd = target.selectionEnd;
+    let text = target.value;
+
+    function selectCurrentParenthesisBlock(OPEN, CLOSE) {
+        if (selectionStart !== selectionEnd) return false;
+
+        // Find opening parenthesis around current cursor
+        const before = text.substring(0, selectionStart);
+        let beforeParen = before.lastIndexOf(OPEN);
+        if (beforeParen == -1) return false;
+        let beforeParenClose = before.lastIndexOf(CLOSE);
+        while (beforeParenClose !== -1 && beforeParenClose > beforeParen) {
+            beforeParen = before.lastIndexOf(OPEN, beforeParen - 1);
+            beforeParenClose = before.lastIndexOf(CLOSE, beforeParenClose - 1);
+        }
+
+        // Find closing parenthesis around current cursor
+        const after = text.substring(selectionStart);
+        let afterParen = after.indexOf(CLOSE);
+        if (afterParen == -1) return false;
+        let afterParenOpen = after.indexOf(OPEN);
+        while (afterParenOpen !== -1 && afterParen > afterParenOpen) {
+            afterParen = after.indexOf(CLOSE, afterParen + 1);
+            afterParenOpen = after.indexOf(OPEN, afterParenOpen + 1);
+        }
+        if (beforeParen === -1 || afterParen === -1) return false;
+
+        // Set the selection to the text between the parenthesis
+        const parenContent = text.substring(beforeParen + 1, selectionStart + afterParen);
+        const lastColon = parenContent.lastIndexOf(":");
+        selectionStart = beforeParen + 1;
+        selectionEnd = selectionStart + lastColon;
+        target.setSelectionRange(selectionStart, selectionEnd);
+        return true;
+    }
+
+    function selectCurrentWord() {
+        if (selectionStart !== selectionEnd) return false;
+        const delimiters = opts.keyedit_delimiters + " \r\n\t";
+
+        // seek backward until to find beggining
+        while (!delimiters.includes(text[selectionStart - 1]) && selectionStart > 0) {
+            selectionStart--;
+        }
+
+        // seek forward to find end
+        while (!delimiters.includes(text[selectionEnd]) && selectionEnd < text.length) {
+            selectionEnd++;
+        }
+
+        target.setSelectionRange(selectionStart, selectionEnd);
+        return true;
+    }
+
+    // If the user hasn't selected anything, let's select their current parenthesis block or word
+    if (!selectCurrentParenthesisBlock('<', '>') && !selectCurrentParenthesisBlock('(', ')')) {
+        selectCurrentWord();
+    }
+
+    event.preventDefault();
+
+    var closeCharacter = ')';
+    var delta = opts.keyedit_precision_attention;
+
+    if (selectionStart > 0 && text[selectionStart - 1] == '<') {
+        closeCharacter = '>';
+        delta = opts.keyedit_precision_extra;
+    } else if (selectionStart == 0 || text[selectionStart - 1] != "(") {
+
+        // do not include spaces at the end
+        while (selectionEnd > selectionStart && text[selectionEnd - 1] == ' ') {
+            selectionEnd -= 1;
+        }
+        if (selectionStart == selectionEnd) {
+            return;
+        }
+
+        text = text.slice(0, selectionStart) + "(" + text.slice(selectionStart, selectionEnd) + ":1.0)" + text.slice(selectionEnd);
+
+        selectionStart += 1;
+        selectionEnd += 1;
+    }
+
+    var end = text.slice(selectionEnd + 1).indexOf(closeCharacter) + 1;
+    var weight = parseFloat(text.slice(selectionEnd + 1, selectionEnd + 1 + end));
+    if (isNaN(weight)) return;
+
+    weight += isPlus ? delta : -delta;
+    weight = parseFloat(weight.toPrecision(12));
+    if (String(weight).length == 1) weight += ".0";
+
+    if (closeCharacter == ')' && weight == 1) {
+        text = text.slice(0, selectionStart - 1) + text.slice(selectionStart, selectionEnd) + text.slice(selectionEnd + 5);
+        selectionStart--;
+        selectionEnd--;
+    } else {
+        text = text.slice(0, selectionEnd + 1) + weight + text.slice(selectionEnd + 1 + end - 1);
+    }
+
+    target.focus();
+    target.value = text;
+    target.selectionStart = selectionStart;
+    target.selectionEnd = selectionEnd;
+
+    updateInput(target);
+}
+
+addEventListener('keydown', (event) => {
+    keyupEditAttention(event);
+});

javascript/extensions.js

@@ -1,35 +1,74 @@
-
-function extensions_apply(_, _){
-    disable = []
-    update = []
-    gradioApp().querySelectorAll('#extensions input[type="checkbox"]').forEach(function(x){
-        if(x.name.startsWith("enable_") && ! x.checked)
-            disable.push(x.name.substr(7))
-
-        if(x.name.startsWith("update_") && x.checked)
-            update.push(x.name.substr(7))
-    })
-
-    restart_reload()
-
-    return [JSON.stringify(disable), JSON.stringify(update)]
-}
-
-function extensions_check(){
-    gradioApp().querySelectorAll('#extensions .extension_status').forEach(function(x){
-        x.innerHTML = "Loading..."
-    })
-
-    return []
-}
-
-function install_extension_from_index(button, url){
-    button.disabled = "disabled"
-    button.value = "Installing..."
-
-    textarea = gradioApp().querySelector('#extension_to_install textarea')
-    textarea.value = url
-    updateInput(textarea)
-
-    gradioApp().querySelector('#install_extension_button').click()
-}
+
+function extensions_apply(_disabled_list, _update_list, disable_all) {
+    var disable = [];
+    var update = [];
+
+    gradioApp().querySelectorAll('#extensions input[type="checkbox"]').forEach(function(x) {
+        if (x.name.startsWith("enable_") && !x.checked) {
+            disable.push(x.name.substring(7));
+        }
+
+        if (x.name.startsWith("update_") && x.checked) {
+            update.push(x.name.substring(7));
+        }
+    });
+
+    restart_reload();
+
+    return [JSON.stringify(disable), JSON.stringify(update), disable_all];
+}
+
+function extensions_check() {
+    var disable = [];
+
+    gradioApp().querySelectorAll('#extensions input[type="checkbox"]').forEach(function(x) {
+        if (x.name.startsWith("enable_") && !x.checked) {
+            disable.push(x.name.substring(7));
+        }
+    });
+
+    gradioApp().querySelectorAll('#extensions .extension_status').forEach(function(x) {
+        x.innerHTML = "Loading...";
+    });
+
+
+    var id = randomId();
+    requestProgress(id, gradioApp().getElementById('extensions_installed_top'), null, function() {
+
+    });
+
+    return [id, JSON.stringify(disable)];
+}
+
+function install_extension_from_index(button, url) {
+    button.disabled = "disabled";
+    button.value = "Installing...";
+
+    var textarea = gradioApp().querySelector('#extension_to_install textarea');
+    textarea.value = url;
+    updateInput(textarea);
+
+    gradioApp().querySelector('#install_extension_button').click();
+}
+
+function config_state_confirm_restore(_, config_state_name, config_restore_type) {
+    if (config_state_name == "Current") {
+        return [false, config_state_name, config_restore_type];
+    }
+    let restored = "";
+    if (config_restore_type == "extensions") {
+        restored = "all saved extension versions";
+    } else if (config_restore_type == "webui") {
+        restored = "the webui version";
+    } else {
+        restored = "the webui version and all saved extension versions";
+    }
+    let confirmed = confirm("Are you sure you want to restore from this state?\nThis will reset " + restored + ".");
+    if (confirmed) {
+        restart_reload();
+        gradioApp().querySelectorAll('#extensions .extension_status').forEach(function(x) {
+            x.innerHTML = "Loading...";
+        });
+    }
+    return [confirmed, config_state_name, config_restore_type];
+}

javascript/extraNetworks.js

@@ -1,69 +1,265 @@
-
-function setupExtraNetworksForTab(tabname){
-    gradioApp().querySelector('#'+tabname+'_extra_tabs').classList.add('extra-networks')
-
-    var tabs = gradioApp().querySelector('#'+tabname+'_extra_tabs > div')
-    var search = gradioApp().querySelector('#'+tabname+'_extra_search textarea')
-    var refresh = gradioApp().getElementById(tabname+'_extra_refresh')
-    var close = gradioApp().getElementById(tabname+'_extra_close')
-
-    search.classList.add('search')
-    tabs.appendChild(search)
-    tabs.appendChild(refresh)
-    tabs.appendChild(close)
-
-    search.addEventListener("input", function(evt){
-        searchTerm = search.value.toLowerCase()
-
-        gradioApp().querySelectorAll('#'+tabname+'_extra_tabs div.card').forEach(function(elem){
-            text = elem.querySelector('.name').textContent.toLowerCase()
-            elem.style.display = text.indexOf(searchTerm) == -1 ? "none" : ""
-        })
-    });
-}
-
-var activePromptTextarea = {};
-
-function setupExtraNetworks(){
-    setupExtraNetworksForTab('txt2img')
-    setupExtraNetworksForTab('img2img')
-
-    function registerPrompt(tabname, id){
-        var textarea = gradioApp().querySelector("#" + id + " > label > textarea");
-
-        if (! activePromptTextarea[tabname]){
-            activePromptTextarea[tabname] = textarea
-        }
-
-		textarea.addEventListener("focus", function(){
-            activePromptTextarea[tabname] = textarea;
-		});
-    }
-
-    registerPrompt('txt2img', 'txt2img_prompt')
-    registerPrompt('txt2img', 'txt2img_neg_prompt')
-    registerPrompt('img2img', 'img2img_prompt')
-    registerPrompt('img2img', 'img2img_neg_prompt')
-}
-
-onUiLoaded(setupExtraNetworks)
-
-function cardClicked(tabname, textToAdd, allowNegativePrompt){
-    var textarea = allowNegativePrompt ? activePromptTextarea[tabname] : gradioApp().querySelector("#" + tabname + "_prompt > label > textarea")
-
-    textarea.value = textarea.value + " " + textToAdd
-    updateInput(textarea)
-}
-
-function saveCardPreview(event, tabname, filename){
-    var textarea = gradioApp().querySelector("#" + tabname + '_preview_filename  > label > textarea')
-    var button = gradioApp().getElementById(tabname + '_save_preview')
-
-    textarea.value = filename
-    updateInput(textarea)
-
-    button.click()
-
-    event.stopPropagation()
-    event.preventDefault()
-}
+function setupExtraNetworksForTab(tabname) {
+    gradioApp().querySelector('#' + tabname + '_extra_tabs').classList.add('extra-networks');
+
+    var tabs = gradioApp().querySelector('#' + tabname + '_extra_tabs > div');
+    var search = gradioApp().querySelector('#' + tabname + '_extra_search textarea');
+    var sort = gradioApp().getElementById(tabname + '_extra_sort');
+    var sortOrder = gradioApp().getElementById(tabname + '_extra_sortorder');
+    var refresh = gradioApp().getElementById(tabname + '_extra_refresh');
+
+    search.classList.add('search');
+    sort.classList.add('sort');
+    sortOrder.classList.add('sortorder');
+    sort.dataset.sortkey = 'sortDefault';
+    tabs.appendChild(search);
+    tabs.appendChild(sort);
+    tabs.appendChild(sortOrder);
+    tabs.appendChild(refresh);
+
+    var applyFilter = function() {
+        var searchTerm = search.value.toLowerCase();
+
+        gradioApp().querySelectorAll('#' + tabname + '_extra_tabs div.card').forEach(function(elem) {
+            var searchOnly = elem.querySelector('.search_only');
+            var text = elem.querySelector('.name').textContent.toLowerCase() + " " + elem.querySelector('.search_term').textContent.toLowerCase();
+
+            var visible = text.indexOf(searchTerm) != -1;
+
+            if (searchOnly && searchTerm.length < 4) {
+                visible = false;
+            }
+
+            elem.style.display = visible ? "" : "none";
+        });
+    };
+
+    var applySort = function() {
+        var reverse = sortOrder.classList.contains("sortReverse");
+        var sortKey = sort.querySelector("input").value.toLowerCase().replace("sort", "").replaceAll(" ", "_").replace(/_+$/, "").trim();
+        sortKey = sortKey ? "sort" + sortKey.charAt(0).toUpperCase() + sortKey.slice(1) : "";
+        var sortKeyStore = sortKey ? sortKey + (reverse ? "Reverse" : "") : "";
+        if (!sortKey || sortKeyStore == sort.dataset.sortkey) {
+            return;
+        }
+
+        sort.dataset.sortkey = sortKeyStore;
+
+        var cards = gradioApp().querySelectorAll('#' + tabname + '_extra_tabs div.card');
+        cards.forEach(function(card) {
+            card.originalParentElement = card.parentElement;
+        });
+        var sortedCards = Array.from(cards);
+        sortedCards.sort(function(cardA, cardB) {
+            var a = cardA.dataset[sortKey];
+            var b = cardB.dataset[sortKey];
+            if (!isNaN(a) && !isNaN(b)) {
+                return parseInt(a) - parseInt(b);
+            }
+
+            return (a < b ? -1 : (a > b ? 1 : 0));
+        });
+        if (reverse) {
+            sortedCards.reverse();
+        }
+        cards.forEach(function(card) {
+            card.remove();
+        });
+        sortedCards.forEach(function(card) {
+            card.originalParentElement.appendChild(card);
+        });
+    };
+
+    search.addEventListener("input", applyFilter);
+    applyFilter();
+    ["change", "blur", "click"].forEach(function(evt) {
+        sort.querySelector("input").addEventListener(evt, applySort);
+    });
+    sortOrder.addEventListener("click", function() {
+        sortOrder.classList.toggle("sortReverse");
+        applySort();
+    });
+
+    extraNetworksApplyFilter[tabname] = applyFilter;
+}
+
+function applyExtraNetworkFilter(tabname) {
+    setTimeout(extraNetworksApplyFilter[tabname], 1);
+}
+
+var extraNetworksApplyFilter = {};
+var activePromptTextarea = {};
+
+function setupExtraNetworks() {
+    setupExtraNetworksForTab('txt2img');
+    setupExtraNetworksForTab('img2img');
+
+    function registerPrompt(tabname, id) {
+        var textarea = gradioApp().querySelector("#" + id + " > label > textarea");
+
+        if (!activePromptTextarea[tabname]) {
+            activePromptTextarea[tabname] = textarea;
+        }
+
+        textarea.addEventListener("focus", function() {
+            activePromptTextarea[tabname] = textarea;
+        });
+    }
+
+    registerPrompt('txt2img', 'txt2img_prompt');
+    registerPrompt('txt2img', 'txt2img_neg_prompt');
+    registerPrompt('img2img', 'img2img_prompt');
+    registerPrompt('img2img', 'img2img_neg_prompt');
+}
+
+onUiLoaded(setupExtraNetworks);
+
+var re_extranet = /<([^:]+:[^:]+):[\d.]+>/;
+var re_extranet_g = /\s+<([^:]+:[^:]+):[\d.]+>/g;
+
+function tryToRemoveExtraNetworkFromPrompt(textarea, text) {
+    var m = text.match(re_extranet);
+    var replaced = false;
+    var newTextareaText;
+    if (m) {
+        var partToSearch = m[1];
+        newTextareaText = textarea.value.replaceAll(re_extranet_g, function(found) {
+            m = found.match(re_extranet);
+            if (m[1] == partToSearch) {
+                replaced = true;
+                return "";
+            }
+            return found;
+        });
+    } else {
+        newTextareaText = textarea.value.replaceAll(new RegExp(text, "g"), function(found) {
+            if (found == text) {
+                replaced = true;
+                return "";
+            }
+            return found;
+        });
+    }
+
+    if (replaced) {
+        textarea.value = newTextareaText;
+        return true;
+    }
+
+    return false;
+}
+
+function cardClicked(tabname, textToAdd, allowNegativePrompt) {
+    var textarea = allowNegativePrompt ? activePromptTextarea[tabname] : gradioApp().querySelector("#" + tabname + "_prompt > label > textarea");
+
+    if (!tryToRemoveExtraNetworkFromPrompt(textarea, textToAdd)) {
+        textarea.value = textarea.value + opts.extra_networks_add_text_separator + textToAdd;
+    }
+
+    updateInput(textarea);
+}
+
+function saveCardPreview(event, tabname, filename) {
+    var textarea = gradioApp().querySelector("#" + tabname + '_preview_filename  > label > textarea');
+    var button = gradioApp().getElementById(tabname + '_save_preview');
+
+    textarea.value = filename;
+    updateInput(textarea);
+
+    button.click();
+
+    event.stopPropagation();
+    event.preventDefault();
+}
+
+function extraNetworksSearchButton(tabs_id, event) {
+    var searchTextarea = gradioApp().querySelector("#" + tabs_id + ' > div > textarea');
+    var button = event.target;
+    var text = button.classList.contains("search-all") ? "" : button.textContent.trim();
+
+    searchTextarea.value = text;
+    updateInput(searchTextarea);
+}
+
+var globalPopup = null;
+var globalPopupInner = null;
+function popup(contents) {
+    if (!globalPopup) {
+        globalPopup = document.createElement('div');
+        globalPopup.onclick = function() {
+            globalPopup.style.display = "none";
+        };
+        globalPopup.classList.add('global-popup');
+
+        var close = document.createElement('div');
+        close.classList.add('global-popup-close');
+        close.onclick = function() {
+            globalPopup.style.display = "none";
+        };
+        close.title = "Close";
+        globalPopup.appendChild(close);
+
+        globalPopupInner = document.createElement('div');
+        globalPopupInner.onclick = function(event) {
+            event.stopPropagation(); return false;
+        };
+        globalPopupInner.classList.add('global-popup-inner');
+        globalPopup.appendChild(globalPopupInner);
+
+        gradioApp().appendChild(globalPopup);
+    }
+
+    globalPopupInner.innerHTML = '';
+    globalPopupInner.appendChild(contents);
+
+    globalPopup.style.display = "flex";
+}
+
+function extraNetworksShowMetadata(text) {
+    var elem = document.createElement('pre');
+    elem.classList.add('popup-metadata');
+    elem.textContent = text;
+
+    popup(elem);
+}
+
+function requestGet(url, data, handler, errorHandler) {
+    var xhr = new XMLHttpRequest();
+    var args = Object.keys(data).map(function(k) {
+        return encodeURIComponent(k) + '=' + encodeURIComponent(data[k]);
+    }).join('&');
+    xhr.open("GET", url + "?" + args, true);
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState === 4) {
+            if (xhr.status === 200) {
+                try {
+                    var js = JSON.parse(xhr.responseText);
+                    handler(js);
+                } catch (error) {
+                    console.error(error);
+                    errorHandler();
+                }
+            } else {
+                errorHandler();
+            }
+        }
+    };
+    var js = JSON.stringify(data);
+    xhr.send(js);
+}
+
+function extraNetworksRequestMetadata(event, extraPage, cardName) {
+    var showError = function() {
+        extraNetworksShowMetadata("there was an error getting metadata");
+    };
+
+    requestGet("./sd_extra_networks/metadata", {page: extraPage, item: cardName}, function(data) {
+        if (data && data.metadata) {
+            extraNetworksShowMetadata(data.metadata);
+        } else {
+            showError();
+        }
+    }, showError);
+
+    event.stopPropagation();
+}

javascript/generationParams.js

@@ -1,33 +1,35 @@
 // attaches listeners to the txt2img and img2img galleries to update displayed generation param text when the image changes
 
 let txt2img_gallery, img2img_gallery, modal = undefined;
-onUiUpdate(function(){
-	if (!txt2img_gallery) {
-		txt2img_gallery = attachGalleryListeners("txt2img")
-	}
-	if (!img2img_gallery) {
-		img2img_gallery = attachGalleryListeners("img2img")
-	}
-	if (!modal) {
-		modal = gradioApp().getElementById('lightboxModal')
-		modalObserver.observe(modal,  { attributes : true, attributeFilter : ['style'] });
-	}
+onAfterUiUpdate(function() {
+    if (!txt2img_gallery) {
+        txt2img_gallery = attachGalleryListeners("txt2img");
+    }
+    if (!img2img_gallery) {
+        img2img_gallery = attachGalleryListeners("img2img");
+    }
+    if (!modal) {
+        modal = gradioApp().getElementById('lightboxModal');
+        modalObserver.observe(modal, {attributes: true, attributeFilter: ['style']});
+    }
 });
 
 let modalObserver = new MutationObserver(function(mutations) {
-	mutations.forEach(function(mutationRecord) {
-		let selectedTab = gradioApp().querySelector('#tabs div button.bg-white')?.innerText
-		if (mutationRecord.target.style.display === 'none' && selectedTab === 'txt2img' || selectedTab === 'img2img')
-			gradioApp().getElementById(selectedTab+"_generation_info_button").click()
-	});
+    mutations.forEach(function(mutationRecord) {
+        let selectedTab = gradioApp().querySelector('#tabs div button.selected')?.innerText;
+        if (mutationRecord.target.style.display === 'none' && (selectedTab === 'txt2img' || selectedTab === 'img2img')) {
+            gradioApp().getElementById(selectedTab + "_generation_info_button")?.click();
+        }
+    });
 });
 
 function attachGalleryListeners(tab_name) {
-	gallery = gradioApp().querySelector('#'+tab_name+'_gallery')
-	gallery?.addEventListener('click', () => gradioApp().getElementById(tab_name+"_generation_info_button").click());
-	gallery?.addEventListener('keydown', (e) => {
-		if (e.keyCode == 37 || e.keyCode == 39) // left or right arrow
-			gradioApp().getElementById(tab_name+"_generation_info_button").click()
-	});
-	return gallery;
+    var gallery = gradioApp().querySelector('#' + tab_name + '_gallery');
+    gallery?.addEventListener('click', () => gradioApp().getElementById(tab_name + "_generation_info_button").click());
+    gallery?.addEventListener('keydown', (e) => {
+        if (e.keyCode == 37 || e.keyCode == 39) { // left or right arrow
+            gradioApp().getElementById(tab_name + "_generation_info_button").click();
+        }
+    });
+    return gallery;
 }

javascript/hints.js

@@ -1,15 +1,17 @@
 // mouseover tooltips for various UI elements
 
-titles = {
+var titles = {
     "Sampling steps": "How many times to improve the generated image iteratively; higher values take longer; very low values can produce bad results",
     "Sampling method": "Which algorithm to use to produce the image",
-	"GFPGAN": "Restore low quality faces using GFPGAN neural network",
-	"Euler a": "Euler Ancestral - very creative, each can get a completely different picture depending on step count, setting steps higher than 30-40 does not help",
-	"DDIM": "Denoising Diffusion Implicit Models - best at inpainting",
-	"DPM adaptive": "Ignores step count - uses a number of steps determined by the CFG and resolution", 
+    "GFPGAN": "Restore low quality faces using GFPGAN neural network",
+    "Euler a": "Euler Ancestral - very creative, each can get a completely different picture depending on step count, setting steps higher than 30-40 does not help",
+    "DDIM": "Denoising Diffusion Implicit Models - best at inpainting",
+    "UniPC": "Unified Predictor-Corrector Framework for Fast Sampling of Diffusion Models",
+    "DPM adaptive": "Ignores step count - uses a number of steps determined by the CFG and resolution",
 
-	"Batch count": "How many batches of images to create",
-	"Batch size": "How many image to create in a single batch",
+    "\u{1F4D0}": "Auto detect size from img2img",
+    "Batch count": "How many batches of images to create (has no impact on generation performance or VRAM usage)",
+    "Batch size": "How many image to create in a single batch (increases generation performance at cost of higher VRAM usage)",
     "CFG Scale": "Classifier Free Guidance Scale - how strongly the image should conform to prompt - lower values produce more creative results",
     "Seed": "A value that determines the output of random number generator - if you create an image with same parameters and seed as another image, you'll get the same result",
     "\u{1f3b2}\ufe0f": "Set seed to -1, which will cause a new random number to be used every time",
@@ -17,11 +19,11 @@ titles = {
     "\u2199\ufe0f": "Read generation parameters from prompt or last generation if prompt is empty into user interface.",
     "\u{1f4c2}": "Open images output directory",
     "\u{1f4be}": "Save style",
-    "\U0001F5D1": "Clear prompt",
+    "\u{1f5d1}\ufe0f": "Clear prompt",
     "\u{1f4cb}": "Apply selected styles to current prompt",
     "\u{1f4d2}": "Paste available values into the field",
-    "\u{1f3b4}": "Show extra networks",
-
+    "\u{1f3b4}": "Show/hide extra networks",
+    "\u{1f300}": "Restore progress",
 
     "Inpaint a part of image": "Draw a mask over an image, and the script will regenerate the masked area with content according to prompt",
     "SD upscale": "Upscale image normally, split result into tiles, improve each tile using img2img, merge whole image back",
@@ -39,7 +41,6 @@ titles = {
     "Inpaint at full resolution": "Upscale masked region to target resolution, do inpainting, downscale back and paste into original image",
 
     "Denoising strength": "Determines how little respect the algorithm should have for image's content. At 0, nothing will change, and at 1 you'll get an unrelated image. With values below 1.0, processing will take less steps than the Sampling Steps slider specifies.",
-    "Denoising strength change factor": "In loopback mode, on each loop the denoising strength is multiplied by this value. <1 means decreasing variety so your sequence will converge on a fixed picture. >1 means increasing variety so your sequence will become more and more chaotic.",
 
     "Skip": "Stop processing current image and continue processing.",
     "Interrupt": "Stop processing images and return any results accumulated so far.",
@@ -50,7 +51,7 @@ titles = {
 
     "None": "Do not do anything special",
     "Prompt matrix": "Separate prompts into parts using vertical pipe character (|) and the script will create a picture for every combination of them (except for the first part, which will be present in all combinations)",
-    "X/Y plot": "Create a grid where images will have different parameters. Use inputs below to specify which parameters will be shared by columns and rows",
+    "X/Y/Z plot": "Create grid(s) where images will have different parameters. Use inputs below to specify which parameters will be shared by columns and rows",
     "Custom code": "Run Python code. Advanced user only. Must run program with --allow-code for this to work",
 
     "Prompt S/R": "Separate a list of words with commas, and the first word will be used as a keyword: script will search for this word in the prompt, and replace it with others",
@@ -66,12 +67,14 @@ titles = {
 
     "Interrogate": "Reconstruct prompt from existing image and put it into the prompt field.",
 
-    "Images filename pattern": "Use following tags to define how filenames for images are chosen: [steps], [cfg], [prompt], [prompt_no_styles], [prompt_spaces], [width], [height], [styles], [sampler], [seed], [model_hash], [model_name], [prompt_words], [date], [datetime], [datetime<Format>], [datetime<Format><Time Zone>], [job_timestamp]; leave empty for default.",
-    "Directory name pattern": "Use following tags to define how subdirectories for images and grids are chosen: [steps], [cfg], [prompt], [prompt_no_styles], [prompt_spaces], [width], [height], [styles], [sampler], [seed], [model_hash], [model_name], [prompt_words], [date], [datetime], [datetime<Format>], [datetime<Format><Time Zone>], [job_timestamp]; leave empty for default.",
+    "Images filename pattern": "Use tags like [seed] and [date] to define how filenames for images are chosen. Leave empty for default.",
+    "Directory name pattern": "Use tags like [seed] and [date] to define how subdirectories for images and grids are chosen. Leave empty for default.",
     "Max prompt words": "Set the maximum number of words to be used in the [prompt_words] option; ATTENTION: If the words are too long, they may exceed the maximum length of the file path that the system can handle",
 
-    "Loopback": "Process an image, use it as an input, repeat.",
-    "Loops": "How many times to repeat processing an image and using it as input for the next iteration",
+    "Loopback": "Performs img2img processing multiple times. Output images are used as input for the next loop.",
+    "Loops": "How many times to process an image. Each output is used as the input of the next loop. If set to 1, behavior will be as if this script were not used.",
+    "Final denoising strength": "The denoising strength for the final loop of each image in the batch.",
+    "Denoising strength curve": "The denoising curve controls the rate of denoising strength change each loop. Aggressive: Most of the change will happen towards the start of the loops. Linear: Change will be constant through all loops. Lazy: Most of the change will happen towards the end of the loops.",
 
     "Style 1": "Style to apply; styles have components for both positive and negative prompts and apply to both",
     "Style 2": "Style to apply; styles have components for both positive and negative prompts and apply to both",
@@ -84,7 +87,6 @@ titles = {
     "vram": "Torch active: Peak amount of VRAM used by Torch during generation, excluding cached data.\nTorch reserved: Peak amount of VRAM allocated by Torch, including all active and cached data.\nSys VRAM: Peak amount of VRAM allocation across all applications / total GPU VRAM (peak utilization%).",
 
     "Eta noise seed delta": "If this values is non-zero, it will be added to seed and used to initialize RNG for noises when using samplers with Eta. You can use this to produce even more variation of images, or you can use this to match images of other software if you know what you are doing.",
-    "Do not add watermark to images": "If this option is enabled, watermark will not be added to created images. Warning: if you do not add watermark, you may be behaving in an unethical manner.",
 
     "Filename word regex": "This regular expression will be used extract words from filename, and they will be joined using the option below into label text used for training. Leave empty to keep filename text as it is.",
     "Filename join string": "This string will be used to join split words into a single line if the option above is enabled.",
@@ -95,7 +97,7 @@ titles = {
     "Add difference": "Result = A + (B - C) * M",
     "No interpolation": "Result = A",
 
-	"Initialization text": "If the number of tokens is more than the number of vectors, some may be skipped.\nLeave the textbox empty to start with zeroed out vectors",
+    "Initialization text": "If the number of tokens is more than the number of vectors, some may be skipped.\nLeave the textbox empty to start with zeroed out vectors",
     "Learning rate": "How fast should training go. Low values will take longer to train, high values may fail to converge (not generate accurate results) and/or may break the embedding (This has happened if you see Loss: nan in the training info textbox. If this happens, you need to manually restore your embedding from an older not-broken backup).\n\nYou can set a single numeric value, or multiple learning rates using the syntax:\n\n   rate_1:max_steps_1, rate_2:max_steps_2, ...\n\nEG:   0.005:100, 1e-3:1000, 1e-5\n\nWill train with rate of 0.005 for first 100 steps, then 1e-3 until 1000 steps, then 1e-5 for all remaining steps.",
 
     "Clip skip": "Early stopping parameter for CLIP model; 1 is stop at last layer as usual, 2 is stop at penultimate layer, etc.",
@@ -110,37 +112,84 @@ titles = {
     "Resize height to": "Resizes image to this height. If 0, height is inferred from either of two nearby sliders.",
     "Multiplier for extra networks": "When adding extra network such as Hypernetwork or Lora to prompt, use this multiplier for it.",
     "Discard weights with matching name": "Regular expression; if weights's name matches it, the weights is not written to the resulting checkpoint. Use ^model_ema to discard EMA weights.",
-    "Extra networks tab order": "Comma-separated list of tab names; tabs listed here will appear in the extra networks UI first and in order lsited."
+    "Extra networks tab order": "Comma-separated list of tab names; tabs listed here will appear in the extra networks UI first and in order lsited.",
+    "Negative Guidance minimum sigma": "Skip negative prompt for steps where image is already mostly denoised; the higher this value, the more skips there will be; provides increased performance in exchange for minor quality reduction."
+};
+
+function updateTooltip(element) {
+    if (element.title) return; // already has a title
+
+    let text = element.textContent;
+    let tooltip = localization[titles[text]] || titles[text];
+
+    if (!tooltip) {
+        let value = element.value;
+        if (value) tooltip = localization[titles[value]] || titles[value];
+    }
+
+    if (!tooltip) {
+        // Gradio dropdown options have `data-value`.
+        let dataValue = element.dataset.value;
+        if (dataValue) tooltip = localization[titles[dataValue]] || titles[dataValue];
+    }
+
+    if (!tooltip) {
+        for (const c of element.classList) {
+            if (c in titles) {
+                tooltip = localization[titles[c]] || titles[c];
+                break;
+            }
+        }
+    }
+
+    if (tooltip) {
+        element.title = tooltip;
+    }
 }
 
+// Nodes to check for adding tooltips.
+const tooltipCheckNodes = new Set();
+// Timer for debouncing tooltip check.
+let tooltipCheckTimer = null;
 
-onUiUpdate(function(){
-	gradioApp().querySelectorAll('span, button, select, p').forEach(function(span){
-		tooltip = titles[span.textContent];
+function processTooltipCheckNodes() {
+    for (const node of tooltipCheckNodes) {
+        updateTooltip(node);
+    }
+    tooltipCheckNodes.clear();
+}
 
-		if(!tooltip){
-		    tooltip = titles[span.value];
-		}
-
-		if(!tooltip){
-			for (const c of span.classList) {
-				if (c in titles) {
-					tooltip = titles[c];
-					break;
-				}
-			}
-		}
-
-		if(tooltip){
-			span.title = tooltip;
-		}
-	})
-
-	gradioApp().querySelectorAll('select').forEach(function(select){
-	    if (select.onchange != null) return;
-
-	    select.onchange = function(){
-            select.title = titles[select.value] || "";
-	    }
-	})
-})
+onUiUpdate(function(mutationRecords) {
+    for (const record of mutationRecords) {
+        if (record.type === "childList" && record.target.classList.contains("options")) {
+            // This smells like a Gradio dropdown menu having changed,
+            // so let's enqueue an update for the input element that shows the current value.
+            let wrap = record.target.parentNode;
+            let input = wrap?.querySelector("input");
+            if (input) {
+                input.title = ""; // So we'll even have a chance to update it.
+                tooltipCheckNodes.add(input);
+            }
+        }
+        for (const node of record.addedNodes) {
+            if (node.nodeType === Node.ELEMENT_NODE && !node.classList.contains("hide")) {
+                if (!node.title) {
+                    if (
+                        node.tagName === "SPAN" ||
+                        node.tagName === "BUTTON" ||
+                        node.tagName === "P" ||
+                        node.tagName === "INPUT" ||
+                        (node.tagName === "LI" && node.classList.contains("item")) // Gradio dropdown item
+                    ) {
+                        tooltipCheckNodes.add(node);
+                    }
+                }
+                node.querySelectorAll('span, button, p').forEach(n => tooltipCheckNodes.add(n));
+            }
+        }
+    }
+    if (tooltipCheckNodes.size) {
+        clearTimeout(tooltipCheckTimer);
+        tooltipCheckTimer = setTimeout(processTooltipCheckNodes, 1000);
+    }
+});

javascript/hires_fix.js

@@ -1,22 +1,18 @@
-
-function setInactive(elem, inactive){
-    if(inactive){
-        elem.classList.add('inactive')
-    } else{
-        elem.classList.remove('inactive')
-    }
-}
-
-function onCalcResolutionHires(enable, width, height, hr_scale, hr_resize_x, hr_resize_y){
-    hrUpscaleBy = gradioApp().getElementById('txt2img_hr_scale')
-    hrResizeX = gradioApp().getElementById('txt2img_hr_resize_x')
-    hrResizeY = gradioApp().getElementById('txt2img_hr_resize_y')
-
-    gradioApp().getElementById('txt2img_hires_fix_row2').style.display = opts.use_old_hires_fix_width_height ? "none" : ""
-
-    setInactive(hrUpscaleBy, opts.use_old_hires_fix_width_height || hr_resize_x > 0 || hr_resize_y > 0)
-    setInactive(hrResizeX, opts.use_old_hires_fix_width_height || hr_resize_x == 0)
-    setInactive(hrResizeY, opts.use_old_hires_fix_width_height || hr_resize_y == 0)
-
-    return [enable, width, height, hr_scale, hr_resize_x, hr_resize_y]
-}
+
+function onCalcResolutionHires(enable, width, height, hr_scale, hr_resize_x, hr_resize_y) {
+    function setInactive(elem, inactive) {
+        elem.classList.toggle('inactive', !!inactive);
+    }
+
+    var hrUpscaleBy = gradioApp().getElementById('txt2img_hr_scale');
+    var hrResizeX = gradioApp().getElementById('txt2img_hr_resize_x');
+    var hrResizeY = gradioApp().getElementById('txt2img_hr_resize_y');
+
+    gradioApp().getElementById('txt2img_hires_fix_row2').style.display = opts.use_old_hires_fix_width_height ? "none" : "";
+
+    setInactive(hrUpscaleBy, opts.use_old_hires_fix_width_height || hr_resize_x > 0 || hr_resize_y > 0);
+    setInactive(hrResizeX, opts.use_old_hires_fix_width_height || hr_resize_x == 0);
+    setInactive(hrResizeY, opts.use_old_hires_fix_width_height || hr_resize_y == 0);
+
+    return [enable, width, height, hr_scale, hr_resize_x, hr_resize_y];
+}

javascript/imageMaskFix.js

@@ -2,20 +2,18 @@
  * temporary fix for https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/668
  * @see https://github.com/gradio-app/gradio/issues/1721
  */
-window.addEventListener( 'resize', () => imageMaskResize());
 function imageMaskResize() {
     const canvases = gradioApp().querySelectorAll('#img2maskimg .touch-none canvas');
-    if ( ! canvases.length ) {
-    canvases_fixed = false;
-    window.removeEventListener( 'resize', imageMaskResize );
-    return;
+    if (!canvases.length) {
+        window.removeEventListener('resize', imageMaskResize);
+        return;
     }
 
     const wrapper = canvases[0].closest('.touch-none');
     const previewImage = wrapper.previousElementSibling;
 
-    if ( ! previewImage.complete ) {
-        previewImage.addEventListener( 'load', () => imageMaskResize());
+    if (!previewImage.complete) {
+        previewImage.addEventListener('load', imageMaskResize);
         return;
     }
 
@@ -24,22 +22,22 @@ function imageMaskResize() {
     const nw = previewImage.naturalWidth;
     const nh = previewImage.naturalHeight;
     const portrait = nh > nw;
-    const factor = portrait;
 
-    const wW = Math.min(w, portrait ? h/nh*nw : w/nw*nw);
-    const wH = Math.min(h, portrait ? h/nh*nh : w/nw*nh);
+    const wW = Math.min(w, portrait ? h / nh * nw : w / nw * nw);
+    const wH = Math.min(h, portrait ? h / nh * nh : w / nw * nh);
 
     wrapper.style.width = `${wW}px`;
     wrapper.style.height = `${wH}px`;
     wrapper.style.left = `0px`;
     wrapper.style.top = `0px`;
 
-    canvases.forEach( c => {
+    canvases.forEach(c => {
         c.style.width = c.style.height = '';
         c.style.maxWidth = '100%';
         c.style.maxHeight = '100%';
         c.style.objectFit = 'contain';
     });
- }
-  
- onUiUpdate(() => imageMaskResize());
+}
+
+onAfterUiUpdate(imageMaskResize);
+window.addEventListener('resize', imageMaskResize);

javascript/imageParams.js

@@ -1,19 +0,0 @@
-window.onload = (function(){
-    window.addEventListener('drop', e => {
-        const target = e.composedPath()[0];
-        const idx = selected_gallery_index();
-        if (target.placeholder.indexOf("Prompt") == -1) return;
-
-        let prompt_target = get_tab_index('tabs') == 1 ? "img2img_prompt_image" : "txt2img_prompt_image";
-
-        e.stopPropagation();
-        e.preventDefault();
-        const imgParent = gradioApp().getElementById(prompt_target);
-        const files = e.dataTransfer.files;
-        const fileInput = imgParent.querySelector('input[type="file"]');
-        if ( fileInput ) {
-            fileInput.files = files;
-            fileInput.dispatchEvent(new Event('change'));
-        }
-    });
-});

javascript/imageviewer.js

@@ -5,24 +5,24 @@ function closeModal() {
 
 function showModal(event) {
     const source = event.target || event.srcElement;
-    const modalImage = gradioApp().getElementById("modalImage")
-    const lb = gradioApp().getElementById("lightboxModal")
-    modalImage.src = source.src
+    const modalImage = gradioApp().getElementById("modalImage");
+    const lb = gradioApp().getElementById("lightboxModal");
+    modalImage.src = source.src;
     if (modalImage.style.display === 'none') {
         lb.style.setProperty('background-image', 'url(' + source.src + ')');
     }
-    lb.style.display = "block";
-    lb.focus()
+    lb.style.display = "flex";
+    lb.focus();
 
-    const tabTxt2Img = gradioApp().getElementById("tab_txt2img")
-    const tabImg2Img = gradioApp().getElementById("tab_img2img")
+    const tabTxt2Img = gradioApp().getElementById("tab_txt2img");
+    const tabImg2Img = gradioApp().getElementById("tab_img2img");
     // show the save button in modal only on txt2img or img2img tabs
     if (tabTxt2Img.style.display != "none" || tabImg2Img.style.display != "none") {
-        gradioApp().getElementById("modal_save").style.display = "inline"
+        gradioApp().getElementById("modal_save").style.display = "inline";
     } else {
-        gradioApp().getElementById("modal_save").style.display = "none"
+        gradioApp().getElementById("modal_save").style.display = "none";
     }
-    event.stopPropagation()
+    event.stopPropagation();
 }
 
 function negmod(n, m) {
@@ -30,157 +30,129 @@ function negmod(n, m) {
 }
 
 function updateOnBackgroundChange() {
-    const modalImage = gradioApp().getElementById("modalImage")
+    const modalImage = gradioApp().getElementById("modalImage");
     if (modalImage && modalImage.offsetParent) {
-        let allcurrentButtons = gradioApp().querySelectorAll(".gallery-item.transition-all.\\!ring-2")
-        let currentButton = null
-        allcurrentButtons.forEach(function(elem) {
-            if (elem.parentElement.offsetParent) {
-                currentButton = elem;
-            }
-        })
+        let currentButton = selected_gallery_button();
 
         if (currentButton?.children?.length > 0 && modalImage.src != currentButton.children[0].src) {
             modalImage.src = currentButton.children[0].src;
             if (modalImage.style.display === 'none') {
-                modal.style.setProperty('background-image', `url(${modalImage.src})`)
+                const modal = gradioApp().getElementById("lightboxModal");
+                modal.style.setProperty('background-image', `url(${modalImage.src})`);
             }
         }
     }
 }
 
 function modalImageSwitch(offset) {
-    var allgalleryButtons = gradioApp().querySelectorAll(".gallery-item.transition-all")
-    var galleryButtons = []
-    allgalleryButtons.forEach(function(elem) {
-        if (elem.parentElement.offsetParent) {
-            galleryButtons.push(elem);
-        }
-    })
+    var galleryButtons = all_gallery_buttons();
 
     if (galleryButtons.length > 1) {
-        var allcurrentButtons = gradioApp().querySelectorAll(".gallery-item.transition-all.\\!ring-2")
-        var currentButton = null
-        allcurrentButtons.forEach(function(elem) {
-            if (elem.parentElement.offsetParent) {
-                currentButton = elem;
-            }
-        })
+        var currentButton = selected_gallery_button();
 
-        var result = -1
+        var result = -1;
         galleryButtons.forEach(function(v, i) {
             if (v == currentButton) {
-                result = i
+                result = i;
             }
-        })
+        });
 
         if (result != -1) {
-            nextButton = galleryButtons[negmod((result + offset), galleryButtons.length)]
-            nextButton.click()
+            var nextButton = galleryButtons[negmod((result + offset), galleryButtons.length)];
+            nextButton.click();
             const modalImage = gradioApp().getElementById("modalImage");
             const modal = gradioApp().getElementById("lightboxModal");
             modalImage.src = nextButton.children[0].src;
             if (modalImage.style.display === 'none') {
-                modal.style.setProperty('background-image', `url(${modalImage.src})`)
+                modal.style.setProperty('background-image', `url(${modalImage.src})`);
             }
             setTimeout(function() {
-                modal.focus()
-            }, 10)
+                modal.focus();
+            }, 10);
         }
     }
 }
 
-function saveImage(){
-    const tabTxt2Img = gradioApp().getElementById("tab_txt2img")
-    const tabImg2Img = gradioApp().getElementById("tab_img2img")
-    const saveTxt2Img = "save_txt2img"
-    const saveImg2Img = "save_img2img"
+function saveImage() {
+    const tabTxt2Img = gradioApp().getElementById("tab_txt2img");
+    const tabImg2Img = gradioApp().getElementById("tab_img2img");
+    const saveTxt2Img = "save_txt2img";
+    const saveImg2Img = "save_img2img";
     if (tabTxt2Img.style.display != "none") {
-        gradioApp().getElementById(saveTxt2Img).click()
+        gradioApp().getElementById(saveTxt2Img).click();
     } else if (tabImg2Img.style.display != "none") {
-        gradioApp().getElementById(saveImg2Img).click()
+        gradioApp().getElementById(saveImg2Img).click();
     } else {
-        console.error("missing implementation for saving modal of this type")
+        console.error("missing implementation for saving modal of this type");
     }
 }
 
 function modalSaveImage(event) {
-    saveImage()
-    event.stopPropagation()
+    saveImage();
+    event.stopPropagation();
 }
 
 function modalNextImage(event) {
-    modalImageSwitch(1)
-    event.stopPropagation()
+    modalImageSwitch(1);
+    event.stopPropagation();
 }
 
 function modalPrevImage(event) {
-    modalImageSwitch(-1)
-    event.stopPropagation()
+    modalImageSwitch(-1);
+    event.stopPropagation();
 }
 
 function modalKeyHandler(event) {
     switch (event.key) {
-        case "s":
-            saveImage()
-            break;
-        case "ArrowLeft":
-            modalPrevImage(event)
-            break;
-        case "ArrowRight":
-            modalNextImage(event)
-            break;
-        case "Escape":
-            closeModal();
-            break;
+    case "s":
+        saveImage();
+        break;
+    case "ArrowLeft":
+        modalPrevImage(event);
+        break;
+    case "ArrowRight":
+        modalNextImage(event);
+        break;
+    case "Escape":
+        closeModal();
+        break;
     }
 }
 
-function showGalleryImage() {
-    setTimeout(function() {
-        fullImg_preview = gradioApp().querySelectorAll('img.w-full.object-contain')
+function setupImageForLightbox(e) {
+    if (e.dataset.modded) {
+        return;
+    }
 
-        if (fullImg_preview != null) {
-            fullImg_preview.forEach(function function_name(e) {
-                if (e.dataset.modded)
-                    return;
-                e.dataset.modded = true;
-                if(e && e.parentElement.tagName == 'DIV'){
-                    e.style.cursor='pointer'
-                    e.style.userSelect='none'
+    e.dataset.modded = true;
+    e.style.cursor = 'pointer';
+    e.style.userSelect = 'none';
 
-                    var isFirefox = isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1
+    var isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;
 
-                    // For Firefox, listening on click first switched to next image then shows the lightbox.
-                    // If you know how to fix this without switching to mousedown event, please.
-                    // For other browsers the event is click to make it possiblr to drag picture.
-                    var event = isFirefox ? 'mousedown' : 'click'
+    // For Firefox, listening on click first switched to next image then shows the lightbox.
+    // If you know how to fix this without switching to mousedown event, please.
+    // For other browsers the event is click to make it possiblr to drag picture.
+    var event = isFirefox ? 'mousedown' : 'click';
 
-                    e.addEventListener(event, function (evt) {
-                        if(!opts.js_modal_lightbox || evt.button != 0) return;
-                        modalZoomSet(gradioApp().getElementById('modalImage'), opts.js_modal_lightbox_initially_zoomed)
-                        evt.preventDefault()
-                        showModal(evt)
-                    }, true);
-                }
-            });
-        }
+    e.addEventListener(event, function(evt) {
+        if (!opts.js_modal_lightbox || evt.button != 0) return;
+
+        modalZoomSet(gradioApp().getElementById('modalImage'), opts.js_modal_lightbox_initially_zoomed);
+        evt.preventDefault();
+        showModal(evt);
+    }, true);
 
-    }, 100);
 }
 
 function modalZoomSet(modalImage, enable) {
-    if (enable) {
-        modalImage.classList.add('modalImageFullscreen');
-    } else {
-        modalImage.classList.remove('modalImageFullscreen');
-    }
+    if (modalImage) modalImage.classList.toggle('modalImageFullscreen', !!enable);
 }
 
 function modalZoomToggle(event) {
-    modalImage = gradioApp().getElementById("modalImage");
-    modalZoomSet(modalImage, !modalImage.classList.contains('modalImageFullscreen'))
-    event.stopPropagation()
+    var modalImage = gradioApp().getElementById("modalImage");
+    modalZoomSet(modalImage, !modalImage.classList.contains('modalImageFullscreen'));
+    event.stopPropagation();
 }
 
 function modalTileImageToggle(event) {
@@ -189,97 +161,94 @@ function modalTileImageToggle(event) {
     const isTiling = modalImage.style.display === 'none';
     if (isTiling) {
         modalImage.style.display = 'block';
-        modal.style.setProperty('background-image', 'none')
+        modal.style.setProperty('background-image', 'none');
     } else {
         modalImage.style.display = 'none';
-        modal.style.setProperty('background-image', `url(${modalImage.src})`)
+        modal.style.setProperty('background-image', `url(${modalImage.src})`);
     }
 
-    event.stopPropagation()
+    event.stopPropagation();
 }
 
-function galleryImageHandler(e) {
-    if (e && e.parentElement.tagName == 'BUTTON') {
-        e.onclick = showGalleryImage;
-    }
-}
-
-onUiUpdate(function() {
-    fullImg_preview = gradioApp().querySelectorAll('img.w-full')
+onAfterUiUpdate(function() {
+    var fullImg_preview = gradioApp().querySelectorAll('.gradio-gallery > div > img');
     if (fullImg_preview != null) {
-        fullImg_preview.forEach(galleryImageHandler);
+        fullImg_preview.forEach(setupImageForLightbox);
     }
     updateOnBackgroundChange();
-})
+});
 
 document.addEventListener("DOMContentLoaded", function() {
-    const modalFragment = document.createDocumentFragment();
-    const modal = document.createElement('div')
+    //const modalFragment = document.createDocumentFragment();
+    const modal = document.createElement('div');
     modal.onclick = closeModal;
     modal.id = "lightboxModal";
-    modal.tabIndex = 0
-    modal.addEventListener('keydown', modalKeyHandler, true)
+    modal.tabIndex = 0;
+    modal.addEventListener('keydown', modalKeyHandler, true);
 
-    const modalControls = document.createElement('div')
+    const modalControls = document.createElement('div');
     modalControls.className = 'modalControls gradio-container';
     modal.append(modalControls);
 
-    const modalZoom = document.createElement('span')
+    const modalZoom = document.createElement('span');
     modalZoom.className = 'modalZoom cursor';
-    modalZoom.innerHTML = '⤡'
-    modalZoom.addEventListener('click', modalZoomToggle, true)
+    modalZoom.innerHTML = '⤡';
+    modalZoom.addEventListener('click', modalZoomToggle, true);
     modalZoom.title = "Toggle zoomed view";
-    modalControls.appendChild(modalZoom)
+    modalControls.appendChild(modalZoom);
 
-    const modalTileImage = document.createElement('span')
+    const modalTileImage = document.createElement('span');
     modalTileImage.className = 'modalTileImage cursor';
-    modalTileImage.innerHTML = '⊞'
-    modalTileImage.addEventListener('click', modalTileImageToggle, true)
+    modalTileImage.innerHTML = '⊞';
+    modalTileImage.addEventListener('click', modalTileImageToggle, true);
     modalTileImage.title = "Preview tiling";
-    modalControls.appendChild(modalTileImage)
+    modalControls.appendChild(modalTileImage);
 
-    const modalSave = document.createElement("span")
-    modalSave.className = "modalSave cursor"
-    modalSave.id = "modal_save"
-    modalSave.innerHTML = "🖫"
-    modalSave.addEventListener("click", modalSaveImage, true)
-    modalSave.title = "Save Image(s)"
-    modalControls.appendChild(modalSave)
+    const modalSave = document.createElement("span");
+    modalSave.className = "modalSave cursor";
+    modalSave.id = "modal_save";
+    modalSave.innerHTML = "🖫";
+    modalSave.addEventListener("click", modalSaveImage, true);
+    modalSave.title = "Save Image(s)";
+    modalControls.appendChild(modalSave);
 
-    const modalClose = document.createElement('span')
+    const modalClose = document.createElement('span');
     modalClose.className = 'modalClose cursor';
-    modalClose.innerHTML = '×'
+    modalClose.innerHTML = '×';
     modalClose.onclick = closeModal;
     modalClose.title = "Close image viewer";
-    modalControls.appendChild(modalClose)
+    modalControls.appendChild(modalClose);
 
-    const modalImage = document.createElement('img')
+    const modalImage = document.createElement('img');
     modalImage.id = 'modalImage';
     modalImage.onclick = closeModal;
-    modalImage.tabIndex = 0
-    modalImage.addEventListener('keydown', modalKeyHandler, true)
-    modal.appendChild(modalImage)
+    modalImage.tabIndex = 0;
+    modalImage.addEventListener('keydown', modalKeyHandler, true);
+    modal.appendChild(modalImage);
 
-    const modalPrev = document.createElement('a')
+    const modalPrev = document.createElement('a');
     modalPrev.className = 'modalPrev';
-    modalPrev.innerHTML = '❮'
-    modalPrev.tabIndex = 0
+    modalPrev.innerHTML = '❮';
+    modalPrev.tabIndex = 0;
     modalPrev.addEventListener('click', modalPrevImage, true);
-    modalPrev.addEventListener('keydown', modalKeyHandler, true)
-    modal.appendChild(modalPrev)
+    modalPrev.addEventListener('keydown', modalKeyHandler, true);
+    modal.appendChild(modalPrev);
 
-    const modalNext = document.createElement('a')
+    const modalNext = document.createElement('a');
     modalNext.className = 'modalNext';
-    modalNext.innerHTML = '❯'
-    modalNext.tabIndex = 0
+    modalNext.innerHTML = '❯';
+    modalNext.tabIndex = 0;
     modalNext.addEventListener('click', modalNextImage, true);
-    modalNext.addEventListener('keydown', modalKeyHandler, true)
+    modalNext.addEventListener('keydown', modalKeyHandler, true);
 
-    modal.appendChild(modalNext)
+    modal.appendChild(modalNext);
 
+    try {
+        gradioApp().appendChild(modal);
+    } catch (e) {
+        gradioApp().body.appendChild(modal);
+    }
 
-    gradioApp().getRootNode().appendChild(modal)
-
-    document.body.appendChild(modalFragment);
+    document.body.appendChild(modal);
 
 });

javascript/imageviewerGamepad.js

@@ -0,0 +1,63 @@
+let gamepads = [];
+
+window.addEventListener('gamepadconnected', (e) => {
+    const index = e.gamepad.index;
+    let isWaiting = false;
+    gamepads[index] = setInterval(async() => {
+        if (!opts.js_modal_lightbox_gamepad || isWaiting) return;
+        const gamepad = navigator.getGamepads()[index];
+        const xValue = gamepad.axes[0];
+        if (xValue <= -0.3) {
+            modalPrevImage(e);
+            isWaiting = true;
+        } else if (xValue >= 0.3) {
+            modalNextImage(e);
+            isWaiting = true;
+        }
+        if (isWaiting) {
+            await sleepUntil(() => {
+                const xValue = navigator.getGamepads()[index].axes[0];
+                if (xValue < 0.3 && xValue > -0.3) {
+                    return true;
+                }
+            }, opts.js_modal_lightbox_gamepad_repeat);
+            isWaiting = false;
+        }
+    }, 10);
+});
+
+window.addEventListener('gamepaddisconnected', (e) => {
+    clearInterval(gamepads[e.gamepad.index]);
+});
+
+/*
+Primarily for vr controller type pointer devices.
+I use the wheel event because there's currently no way to do it properly with web xr.
+ */
+let isScrolling = false;
+window.addEventListener('wheel', (e) => {
+    if (!opts.js_modal_lightbox_gamepad || isScrolling) return;
+    isScrolling = true;
+
+    if (e.deltaX <= -0.6) {
+        modalPrevImage(e);
+    } else if (e.deltaX >= 0.6) {
+        modalNextImage(e);
+    }
+
+    setTimeout(() => {
+        isScrolling = false;
+    }, opts.js_modal_lightbox_gamepad_repeat);
+});
+
+function sleepUntil(f, timeout) {
+    return new Promise((resolve) => {
+        const timeStart = new Date();
+        const wait = setInterval(function() {
+            if (f() || new Date() - timeStart > timeout) {
+                clearInterval(wait);
+                resolve();
+            }
+        }, 20);
+    });
+}

javascript/localization.js

@@ -1,165 +1,176 @@
-
-// localization = {} -- the dict with translations is created by the backend
-
-ignore_ids_for_localization={
-    setting_sd_hypernetwork: 'OPTION',
-    setting_sd_model_checkpoint: 'OPTION',
-    setting_realesrgan_enabled_models: 'OPTION',
-    modelmerger_primary_model_name: 'OPTION',
-    modelmerger_secondary_model_name: 'OPTION',
-    modelmerger_tertiary_model_name: 'OPTION',
-    train_embedding: 'OPTION',
-    train_hypernetwork: 'OPTION',
-    txt2img_styles: 'OPTION',
-    img2img_styles: 'OPTION',
-    setting_random_artist_categories: 'SPAN',
-    setting_face_restoration_model: 'SPAN',
-    setting_realesrgan_enabled_models: 'SPAN',
-    extras_upscaler_1: 'SPAN',
-    extras_upscaler_2: 'SPAN',
-}
-
-re_num = /^[\.\d]+$/
-re_emoji = /[\p{Extended_Pictographic}\u{1F3FB}-\u{1F3FF}\u{1F9B0}-\u{1F9B3}]/u
-
-original_lines = {}
-translated_lines = {}
-
-function textNodesUnder(el){
-    var n, a=[], walk=document.createTreeWalker(el,NodeFilter.SHOW_TEXT,null,false);
-    while(n=walk.nextNode()) a.push(n);
-    return a;
-}
-
-function canBeTranslated(node, text){
-    if(! text) return false;
-    if(! node.parentElement) return false;
-
-    parentType = node.parentElement.nodeName
-    if(parentType=='SCRIPT' || parentType=='STYLE' || parentType=='TEXTAREA') return false;
-
-    if (parentType=='OPTION' || parentType=='SPAN'){
-        pnode = node
-        for(var level=0; level<4; level++){
-            pnode = pnode.parentElement
-            if(! pnode) break;
-
-            if(ignore_ids_for_localization[pnode.id] == parentType) return false;
-        }
-    }
-
-    if(re_num.test(text)) return false;
-    if(re_emoji.test(text)) return false;
-    return true
-}
-
-function getTranslation(text){
-    if(! text) return undefined
-
-    if(translated_lines[text] === undefined){
-        original_lines[text] = 1
-    }
-
-    tl = localization[text]
-    if(tl !== undefined){
-        translated_lines[tl] = 1
-    }
-
-    return tl
-}
-
-function processTextNode(node){
-    text = node.textContent.trim()
-
-    if(! canBeTranslated(node, text)) return
-
-    tl = getTranslation(text)
-    if(tl !== undefined){
-        node.textContent = tl
-    }
-}
-
-function processNode(node){
-    if(node.nodeType == 3){
-        processTextNode(node)
-        return
-    }
-
-    if(node.title){
-        tl = getTranslation(node.title)
-        if(tl !== undefined){
-            node.title = tl
-        }
-    }
-
-    if(node.placeholder){
-        tl = getTranslation(node.placeholder)
-        if(tl !== undefined){
-            node.placeholder = tl
-        }
-    }
-
-    textNodesUnder(node).forEach(function(node){
-        processTextNode(node)
-    })
-}
-
-function dumpTranslations(){
-    dumped = {}
-    if (localization.rtl) {
-        dumped.rtl = true
-    }
-
-    Object.keys(original_lines).forEach(function(text){
-        if(dumped[text] !== undefined)  return
-
-        dumped[text] = localization[text] || text
-    })
-
-    return dumped
-}
-
-onUiUpdate(function(m){
-    m.forEach(function(mutation){
-        mutation.addedNodes.forEach(function(node){
-            processNode(node)
-        })
-    });
-})
-
-
-document.addEventListener("DOMContentLoaded", function() {
-    processNode(gradioApp())
-
-    if (localization.rtl) {  // if the language is from right to left,
-        (new MutationObserver((mutations, observer) => { // wait for the style to load
-            mutations.forEach(mutation => {
-                mutation.addedNodes.forEach(node => {
-                    if (node.tagName === 'STYLE') {
-                        observer.disconnect();
-
-                        for (const x of node.sheet.rules) {  // find all rtl media rules
-                            if (Array.from(x.media || []).includes('rtl')) {
-                                x.media.appendMedium('all');  // enable them
-                            }
-                        }
-                    }
-                })
-            });
-        })).observe(gradioApp(), { childList: true });
-    }
-})
-
-function download_localization() {
-    text = JSON.stringify(dumpTranslations(), null, 4)
-
-    var element = document.createElement('a');
-    element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(text));
-    element.setAttribute('download', "localization.json");
-    element.style.display = 'none';
-    document.body.appendChild(element);
-
-    element.click();
-
-    document.body.removeChild(element);
-}
+
+// localization = {} -- the dict with translations is created by the backend
+
+var ignore_ids_for_localization = {
+    setting_sd_hypernetwork: 'OPTION',
+    setting_sd_model_checkpoint: 'OPTION',
+    modelmerger_primary_model_name: 'OPTION',
+    modelmerger_secondary_model_name: 'OPTION',
+    modelmerger_tertiary_model_name: 'OPTION',
+    train_embedding: 'OPTION',
+    train_hypernetwork: 'OPTION',
+    txt2img_styles: 'OPTION',
+    img2img_styles: 'OPTION',
+    setting_random_artist_categories: 'SPAN',
+    setting_face_restoration_model: 'SPAN',
+    setting_realesrgan_enabled_models: 'SPAN',
+    extras_upscaler_1: 'SPAN',
+    extras_upscaler_2: 'SPAN',
+};
+
+var re_num = /^[.\d]+$/;
+var re_emoji = /[\p{Extended_Pictographic}\u{1F3FB}-\u{1F3FF}\u{1F9B0}-\u{1F9B3}]/u;
+
+var original_lines = {};
+var translated_lines = {};
+
+function hasLocalization() {
+    return window.localization && Object.keys(window.localization).length > 0;
+}
+
+function textNodesUnder(el) {
+    var n, a = [], walk = document.createTreeWalker(el, NodeFilter.SHOW_TEXT, null, false);
+    while ((n = walk.nextNode())) a.push(n);
+    return a;
+}
+
+function canBeTranslated(node, text) {
+    if (!text) return false;
+    if (!node.parentElement) return false;
+
+    var parentType = node.parentElement.nodeName;
+    if (parentType == 'SCRIPT' || parentType == 'STYLE' || parentType == 'TEXTAREA') return false;
+
+    if (parentType == 'OPTION' || parentType == 'SPAN') {
+        var pnode = node;
+        for (var level = 0; level < 4; level++) {
+            pnode = pnode.parentElement;
+            if (!pnode) break;
+
+            if (ignore_ids_for_localization[pnode.id] == parentType) return false;
+        }
+    }
+
+    if (re_num.test(text)) return false;
+    if (re_emoji.test(text)) return false;
+    return true;
+}
+
+function getTranslation(text) {
+    if (!text) return undefined;
+
+    if (translated_lines[text] === undefined) {
+        original_lines[text] = 1;
+    }
+
+    var tl = localization[text];
+    if (tl !== undefined) {
+        translated_lines[tl] = 1;
+    }
+
+    return tl;
+}
+
+function processTextNode(node) {
+    var text = node.textContent.trim();
+
+    if (!canBeTranslated(node, text)) return;
+
+    var tl = getTranslation(text);
+    if (tl !== undefined) {
+        node.textContent = tl;
+    }
+}
+
+function processNode(node) {
+    if (node.nodeType == 3) {
+        processTextNode(node);
+        return;
+    }
+
+    if (node.title) {
+        let tl = getTranslation(node.title);
+        if (tl !== undefined) {
+            node.title = tl;
+        }
+    }
+
+    if (node.placeholder) {
+        let tl = getTranslation(node.placeholder);
+        if (tl !== undefined) {
+            node.placeholder = tl;
+        }
+    }
+
+    textNodesUnder(node).forEach(function(node) {
+        processTextNode(node);
+    });
+}
+
+function dumpTranslations() {
+    if (!hasLocalization()) {
+        // If we don't have any localization,
+        // we will not have traversed the app to find
+        // original_lines, so do that now.
+        processNode(gradioApp());
+    }
+    var dumped = {};
+    if (localization.rtl) {
+        dumped.rtl = true;
+    }
+
+    for (const text in original_lines) {
+        if (dumped[text] !== undefined) continue;
+        dumped[text] = localization[text] || text;
+    }
+
+    return dumped;
+}
+
+function download_localization() {
+    var text = JSON.stringify(dumpTranslations(), null, 4);
+
+    var element = document.createElement('a');
+    element.setAttribute('href', 'data:text/plain;charset=utf-8,' + encodeURIComponent(text));
+    element.setAttribute('download', "localization.json");
+    element.style.display = 'none';
+    document.body.appendChild(element);
+
+    element.click();
+
+    document.body.removeChild(element);
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+    if (!hasLocalization()) {
+        return;
+    }
+
+    onUiUpdate(function(m) {
+        m.forEach(function(mutation) {
+            mutation.addedNodes.forEach(function(node) {
+                processNode(node);
+            });
+        });
+    });
+
+    processNode(gradioApp());
+
+    if (localization.rtl) { // if the language is from right to left,
+        (new MutationObserver((mutations, observer) => { // wait for the style to load
+            mutations.forEach(mutation => {
+                mutation.addedNodes.forEach(node => {
+                    if (node.tagName === 'STYLE') {
+                        observer.disconnect();
+
+                        for (const x of node.sheet.rules) { // find all rtl media rules
+                            if (Array.from(x.media || []).includes('rtl')) {
+                                x.media.appendMedium('all'); // enable them
+                            }
+                        }
+                    }
+                });
+            });
+        })).observe(gradioApp(), {childList: true});
+    }
+});

javascript/notification.js

@@ -2,20 +2,20 @@
 
 let lastHeadImg = null;
 
-notificationButton = null
+let notificationButton = null;
 
-onUiUpdate(function(){
-    if(notificationButton == null){
-        notificationButton = gradioApp().getElementById('request_notifications')
+onAfterUiUpdate(function() {
+    if (notificationButton == null) {
+        notificationButton = gradioApp().getElementById('request_notifications');
 
-        if(notificationButton != null){
-            notificationButton.addEventListener('click', function (evt) {
-                Notification.requestPermission();
-            },true);
+        if (notificationButton != null) {
+            notificationButton.addEventListener('click', () => {
+                void Notification.requestPermission();
+            }, true);
         }
     }
 
-    const galleryPreviews = gradioApp().querySelectorAll('div[id^="tab_"][style*="display: block"] img.h-full.w-full.overflow-hidden');
+    const galleryPreviews = gradioApp().querySelectorAll('div[id^="tab_"][style*="display: block"] div[id$="_results"] .thumbnail-item > img');
 
     if (galleryPreviews == null) return;
 
@@ -42,7 +42,7 @@ onUiUpdate(function(){
         }
     );
 
-    notification.onclick = function(_){
+    notification.onclick = function(_) {
         parent.focus();
         this.close();
     };

javascript/profilerVisualization.js

@@ -0,0 +1,153 @@
+
+function createRow(table, cellName, items) {
+    var tr = document.createElement('tr');
+    var res = [];
+
+    items.forEach(function(x, i) {
+        if (x === undefined) {
+            res.push(null);
+            return;
+        }
+
+        var td = document.createElement(cellName);
+        td.textContent = x;
+        tr.appendChild(td);
+        res.push(td);
+
+        var colspan = 1;
+        for (var n = i + 1; n < items.length; n++) {
+            if (items[n] !== undefined) {
+                break;
+            }
+
+            colspan += 1;
+        }
+
+        if (colspan > 1) {
+            td.colSpan = colspan;
+        }
+    });
+
+    table.appendChild(tr);
+
+    return res;
+}
+
+function showProfile(path, cutoff = 0.05) {
+    requestGet(path, {}, function(data) {
+        var table = document.createElement('table');
+        table.className = 'popup-table';
+
+        data.records['total'] = data.total;
+        var keys = Object.keys(data.records).sort(function(a, b) {
+            return data.records[b] - data.records[a];
+        });
+        var items = keys.map(function(x) {
+            return {key: x, parts: x.split('/'), time: data.records[x]};
+        });
+        var maxLength = items.reduce(function(a, b) {
+            return Math.max(a, b.parts.length);
+        }, 0);
+
+        var cols = createRow(table, 'th', ['record', 'seconds']);
+        cols[0].colSpan = maxLength;
+
+        function arraysEqual(a, b) {
+            return !(a < b || b < a);
+        }
+
+        var addLevel = function(level, parent, hide) {
+            var matching = items.filter(function(x) {
+                return x.parts[level] && !x.parts[level + 1] && arraysEqual(x.parts.slice(0, level), parent);
+            });
+            var sorted = matching.sort(function(a, b) {
+                return b.time - a.time;
+            });
+            var othersTime = 0;
+            var othersList = [];
+            var othersRows = [];
+            var childrenRows = [];
+            sorted.forEach(function(x) {
+                var visible = x.time >= cutoff && !hide;
+
+                var cells = [];
+                for (var i = 0; i < maxLength; i++) {
+                    cells.push(x.parts[i]);
+                }
+                cells.push(x.time.toFixed(3));
+                var cols = createRow(table, 'td', cells);
+                for (i = 0; i < level; i++) {
+                    cols[i].className = 'muted';
+                }
+
+                var tr = cols[0].parentNode;
+                if (!visible) {
+                    tr.classList.add("hidden");
+                }
+
+                if (x.time >= cutoff) {
+                    childrenRows.push(tr);
+                } else {
+                    othersTime += x.time;
+                    othersList.push(x.parts[level]);
+                    othersRows.push(tr);
+                }
+
+                var children = addLevel(level + 1, parent.concat([x.parts[level]]), true);
+                if (children.length > 0) {
+                    var cell = cols[level];
+                    var onclick = function() {
+                        cell.classList.remove("link");
+                        cell.removeEventListener("click", onclick);
+                        children.forEach(function(x) {
+                            x.classList.remove("hidden");
+                        });
+                    };
+                    cell.classList.add("link");
+                    cell.addEventListener("click", onclick);
+                }
+            });
+
+            if (othersTime > 0) {
+                var cells = [];
+                for (var i = 0; i < maxLength; i++) {
+                    cells.push(parent[i]);
+                }
+                cells.push(othersTime.toFixed(3));
+                cells[level] = 'others';
+                var cols = createRow(table, 'td', cells);
+                for (i = 0; i < level; i++) {
+                    cols[i].className = 'muted';
+                }
+
+                var cell = cols[level];
+                var tr = cell.parentNode;
+                var onclick = function() {
+                    tr.classList.add("hidden");
+                    cell.classList.remove("link");
+                    cell.removeEventListener("click", onclick);
+                    othersRows.forEach(function(x) {
+                        x.classList.remove("hidden");
+                    });
+                };
+
+                cell.title = othersList.join(", ");
+                cell.classList.add("link");
+                cell.addEventListener("click", onclick);
+
+                if (hide) {
+                    tr.classList.add("hidden");
+                }
+
+                childrenRows.push(tr);
+            }
+
+            return childrenRows;
+        };
+
+        addLevel(0, []);
+
+        popup(table);
+    });
+}
+

javascript/progressbar.js

@@ -1,95 +1,29 @@
 // code related to showing and updating progressbar shown as the image is being made
 
+function rememberGallerySelection() {
 
-galleries = {}
-storedGallerySelections = {}
-galleryObservers = {}
-
-function rememberGallerySelection(id_gallery){
-    storedGallerySelections[id_gallery] = getGallerySelectedIndex(id_gallery)
 }
 
-function getGallerySelectedIndex(id_gallery){
-    let galleryButtons = gradioApp().querySelectorAll('#'+id_gallery+' .gallery-item')
-    let galleryBtnSelected = gradioApp().querySelector('#'+id_gallery+' .gallery-item.\\!ring-2')
+function getGallerySelectedIndex() {
 
-     let currentlySelectedIndex = -1
-     galleryButtons.forEach(function(v, i){ if(v==galleryBtnSelected) { currentlySelectedIndex = i } })
-
-     return currentlySelectedIndex
 }
 
-// this is a workaround for https://github.com/gradio-app/gradio/issues/2984
-function check_gallery(id_gallery){
-    let gallery = gradioApp().getElementById(id_gallery)
-    // if gallery has no change, no need to setting up observer again.
-    if (gallery && galleries[id_gallery] !== gallery){
-        galleries[id_gallery] = gallery;
-        if(galleryObservers[id_gallery]){
-            galleryObservers[id_gallery].disconnect();
-        }
-
-        storedGallerySelections[id_gallery] = -1
-
-        galleryObservers[id_gallery] = new MutationObserver(function (){
-            let galleryButtons = gradioApp().querySelectorAll('#'+id_gallery+' .gallery-item')
-            let galleryBtnSelected = gradioApp().querySelector('#'+id_gallery+' .gallery-item.\\!ring-2')
-            let currentlySelectedIndex = getGallerySelectedIndex(id_gallery)
-            prevSelectedIndex = storedGallerySelections[id_gallery]
-            storedGallerySelections[id_gallery] = -1
-
-            if (prevSelectedIndex !== -1 && galleryButtons.length>prevSelectedIndex && !galleryBtnSelected) {
-                // automatically re-open previously selected index (if exists)
-                activeElement = gradioApp().activeElement;
-                let scrollX = window.scrollX;
-                let scrollY = window.scrollY;
-
-                galleryButtons[prevSelectedIndex].click();
-                showGalleryImage();
-
-                // When the gallery button is clicked, it gains focus and scrolls itself into view
-                // We need to scroll back to the previous position
-                setTimeout(function (){
-                    window.scrollTo(scrollX, scrollY);
-                }, 50);
-
-                if(activeElement){
-                    // i fought this for about an hour; i don't know why the focus is lost or why this helps recover it
-                    // if someone has a better solution please by all means
-                    setTimeout(function (){
-                        activeElement.focus({
-                            preventScroll: true // Refocus the element that was focused before the gallery was opened without scrolling to it
-                        })
-                    }, 1);
-                }
-            }
-        })
-        galleryObservers[id_gallery].observe( gallery, { childList:true, subtree:false })
-    }
-}
-
-onUiUpdate(function(){
-    check_gallery('txt2img_gallery')
-    check_gallery('img2img_gallery')
-})
-
-function request(url, data, handler, errorHandler){
+function request(url, data, handler, errorHandler) {
     var xhr = new XMLHttpRequest();
-    var url = url;
     xhr.open("POST", url, true);
     xhr.setRequestHeader("Content-Type", "application/json");
-    xhr.onreadystatechange = function () {
+    xhr.onreadystatechange = function() {
         if (xhr.readyState === 4) {
             if (xhr.status === 200) {
                 try {
                     var js = JSON.parse(xhr.responseText);
-                    handler(js)
+                    handler(js);
                 } catch (error) {
                     console.error(error);
-                    errorHandler()
+                    errorHandler();
                 }
-            } else{
-                errorHandler()
+            } else {
+                errorHandler();
             }
         }
     };
@@ -97,147 +31,147 @@ function request(url, data, handler, errorHandler){
     xhr.send(js);
 }
 
-function pad2(x){
-    return x<10 ? '0'+x : x
+function pad2(x) {
+    return x < 10 ? '0' + x : x;
 }
 
-function formatTime(secs){
-    if(secs > 3600){
-        return pad2(Math.floor(secs/60/60)) + ":" + pad2(Math.floor(secs/60)%60) + ":" + pad2(Math.floor(secs)%60)
-    } else if(secs > 60){
-        return pad2(Math.floor(secs/60)) + ":" + pad2(Math.floor(secs)%60)
-    } else{
-        return Math.floor(secs) + "s"
+function formatTime(secs) {
+    if (secs > 3600) {
+        return pad2(Math.floor(secs / 60 / 60)) + ":" + pad2(Math.floor(secs / 60) % 60) + ":" + pad2(Math.floor(secs) % 60);
+    } else if (secs > 60) {
+        return pad2(Math.floor(secs / 60)) + ":" + pad2(Math.floor(secs) % 60);
+    } else {
+        return Math.floor(secs) + "s";
     }
 }
 
-function setTitle(progress){
-    var title = 'Stable Diffusion'
+function setTitle(progress) {
+    var title = 'Stable Diffusion';
 
-    if(opts.show_progress_in_title && progress){
+    if (opts.show_progress_in_title && progress) {
         title = '[' + progress.trim() + '] ' + title;
     }
 
-    if(document.title != title){
-        document.title =  title;
+    if (document.title != title) {
+        document.title = title;
     }
 }
 
 
-function randomId(){
-    return "task(" + Math.random().toString(36).slice(2, 7) + Math.random().toString(36).slice(2, 7) + Math.random().toString(36).slice(2, 7)+")"
+function randomId() {
+    return "task(" + Math.random().toString(36).slice(2, 7) + Math.random().toString(36).slice(2, 7) + Math.random().toString(36).slice(2, 7) + ")";
 }
 
 // starts sending progress requests to "/internal/progress" uri, creating progressbar above progressbarContainer element and
 // preview inside gallery element. Cleans up all created stuff when the task is over and calls atEnd.
 // calls onProgress every time there is a progress update
-function requestProgress(id_task, progressbarContainer, gallery, atEnd, onProgress){
-    var dateStart = new Date()
-    var wasEverActive = false
-    var parentProgressbar = progressbarContainer.parentNode
-    var parentGallery = gallery ? gallery.parentNode : null
+function requestProgress(id_task, progressbarContainer, gallery, atEnd, onProgress, inactivityTimeout = 40) {
+    var dateStart = new Date();
+    var wasEverActive = false;
+    var parentProgressbar = progressbarContainer.parentNode;
+    var parentGallery = gallery ? gallery.parentNode : null;
 
-    var divProgress = document.createElement('div')
-    divProgress.className='progressDiv'
-    divProgress.style.display = opts.show_progressbar ? "" : "none"
-    var divInner = document.createElement('div')
-    divInner.className='progress'
+    var divProgress = document.createElement('div');
+    divProgress.className = 'progressDiv';
+    divProgress.style.display = opts.show_progressbar ? "block" : "none";
+    var divInner = document.createElement('div');
+    divInner.className = 'progress';
 
-    divProgress.appendChild(divInner)
-    parentProgressbar.insertBefore(divProgress, progressbarContainer)
+    divProgress.appendChild(divInner);
+    parentProgressbar.insertBefore(divProgress, progressbarContainer);
 
-    if(parentGallery){
-        var livePreview = document.createElement('div')
-        livePreview.className='livePreview'
-        parentGallery.insertBefore(livePreview, gallery)
+    if (parentGallery) {
+        var livePreview = document.createElement('div');
+        livePreview.className = 'livePreview';
+        parentGallery.insertBefore(livePreview, gallery);
     }
 
-    var removeProgressBar = function(){
-        setTitle("")
-        parentProgressbar.removeChild(divProgress)
-        if(parentGallery) parentGallery.removeChild(livePreview)
-        atEnd()
-    }
+    var removeProgressBar = function() {
+        setTitle("");
+        parentProgressbar.removeChild(divProgress);
+        if (parentGallery) parentGallery.removeChild(livePreview);
+        atEnd();
+    };
 
-    var fun = function(id_task, id_live_preview){
-        request("./internal/progress", {"id_task": id_task, "id_live_preview": id_live_preview}, function(res){
-            if(res.completed){
-                removeProgressBar()
-                return
+    var fun = function(id_task, id_live_preview) {
+        request("./internal/progress", {id_task: id_task, id_live_preview: id_live_preview}, function(res) {
+            if (res.completed) {
+                removeProgressBar();
+                return;
             }
 
-            var rect = progressbarContainer.getBoundingClientRect()
+            var rect = progressbarContainer.getBoundingClientRect();
 
-            if(rect.width){
+            if (rect.width) {
                 divProgress.style.width = rect.width + "px";
             }
 
-            progressText = ""
+            let progressText = "";
 
-            divInner.style.width = ((res.progress || 0) * 100.0) + '%'
-            divInner.style.background = res.progress ? "" : "transparent"
+            divInner.style.width = ((res.progress || 0) * 100.0) + '%';
+            divInner.style.background = res.progress ? "" : "transparent";
 
-            if(res.progress > 0){
-                progressText = ((res.progress || 0) * 100.0).toFixed(0) + '%'
+            if (res.progress > 0) {
+                progressText = ((res.progress || 0) * 100.0).toFixed(0) + '%';
             }
 
-            if(res.eta){
-                progressText += " ETA: " + formatTime(res.eta)
+            if (res.eta) {
+                progressText += " ETA: " + formatTime(res.eta);
             }
 
 
-            setTitle(progressText)
+            setTitle(progressText);
 
-            if(res.textinfo && res.textinfo.indexOf("\n") == -1){
-                progressText = res.textinfo + " " + progressText
+            if (res.textinfo && res.textinfo.indexOf("\n") == -1) {
+                progressText = res.textinfo + " " + progressText;
             }
 
-            divInner.textContent = progressText
+            divInner.textContent = progressText;
 
-            var elapsedFromStart = (new Date() - dateStart) / 1000
+            var elapsedFromStart = (new Date() - dateStart) / 1000;
 
-            if(res.active) wasEverActive = true;
+            if (res.active) wasEverActive = true;
 
-            if(! res.active && wasEverActive){
-                removeProgressBar()
-                return
+            if (!res.active && wasEverActive) {
+                removeProgressBar();
+                return;
             }
 
-            if(elapsedFromStart > 5 && !res.queued && !res.active){
-                removeProgressBar()
-                return
+            if (elapsedFromStart > inactivityTimeout && !res.queued && !res.active) {
+                removeProgressBar();
+                return;
             }
 
 
-            if(res.live_preview && gallery){
-                var rect = gallery.getBoundingClientRect()
-                if(rect.width){
-                    livePreview.style.width = rect.width + "px"
-                    livePreview.style.height = rect.height + "px"
+            if (res.live_preview && gallery) {
+                rect = gallery.getBoundingClientRect();
+                if (rect.width) {
+                    livePreview.style.width = rect.width + "px";
+                    livePreview.style.height = rect.height + "px";
                 }
 
                 var img = new Image();
                 img.onload = function() {
-                    livePreview.appendChild(img)
-                    if(livePreview.childElementCount > 2){
-                        livePreview.removeChild(livePreview.firstElementChild)
+                    livePreview.appendChild(img);
+                    if (livePreview.childElementCount > 2) {
+                        livePreview.removeChild(livePreview.firstElementChild);
                     }
-                }
+                };
                 img.src = res.live_preview;
             }
 
 
-            if(onProgress){
-                onProgress(res)
+            if (onProgress) {
+                onProgress(res);
             }
 
             setTimeout(() => {
                 fun(id_task, res.id_live_preview);
-            }, opts.live_preview_refresh_period || 500)
-        }, function(){
-            removeProgressBar()
-        })
-    }
+            }, opts.live_preview_refresh_period || 500);
+        }, function() {
+            removeProgressBar();
+        });
+    };
 
-    fun(id_task, 0)
+    fun(id_task, 0);
 }

javascript/textualInversion.js

@@ -1,17 +1,17 @@
-
-
-
-function start_training_textual_inversion(){
-    gradioApp().querySelector('#ti_error').innerHTML=''
-
-    var id = randomId()
-    requestProgress(id, gradioApp().getElementById('ti_output'), gradioApp().getElementById('ti_gallery'), function(){}, function(progress){
-        gradioApp().getElementById('ti_progress').innerHTML = progress.textinfo
-    })
-
-    var res = args_to_array(arguments)
-
-    res[0] = id
-
-    return res
-}
+
+
+
+function start_training_textual_inversion() {
+    gradioApp().querySelector('#ti_error').innerHTML = '';
+
+    var id = randomId();
+    requestProgress(id, gradioApp().getElementById('ti_output'), gradioApp().getElementById('ti_gallery'), function() {}, function(progress) {
+        gradioApp().getElementById('ti_progress').innerHTML = progress.textinfo;
+    });
+
+    var res = Array.from(arguments);
+
+    res[0] = id;
+
+    return res;
+}

javascript/token-counters.js

@@ -0,0 +1,83 @@
+let promptTokenCountDebounceTime = 800;
+let promptTokenCountTimeouts = {};
+var promptTokenCountUpdateFunctions = {};
+
+function update_txt2img_tokens(...args) {
+    // Called from Gradio
+    update_token_counter("txt2img_token_button");
+    if (args.length == 2) {
+        return args[0];
+    }
+    return args;
+}
+
+function update_img2img_tokens(...args) {
+    // Called from Gradio
+    update_token_counter("img2img_token_button");
+    if (args.length == 2) {
+        return args[0];
+    }
+    return args;
+}
+
+function update_token_counter(button_id) {
+    if (opts.disable_token_counters) {
+        return;
+    }
+    if (promptTokenCountTimeouts[button_id]) {
+        clearTimeout(promptTokenCountTimeouts[button_id]);
+    }
+    promptTokenCountTimeouts[button_id] = setTimeout(
+        () => gradioApp().getElementById(button_id)?.click(),
+        promptTokenCountDebounceTime,
+    );
+}
+
+
+function recalculatePromptTokens(name) {
+    promptTokenCountUpdateFunctions[name]?.();
+}
+
+function recalculate_prompts_txt2img() {
+    // Called from Gradio
+    recalculatePromptTokens('txt2img_prompt');
+    recalculatePromptTokens('txt2img_neg_prompt');
+    return Array.from(arguments);
+}
+
+function recalculate_prompts_img2img() {
+    // Called from Gradio
+    recalculatePromptTokens('img2img_prompt');
+    recalculatePromptTokens('img2img_neg_prompt');
+    return Array.from(arguments);
+}
+
+function setupTokenCounting(id, id_counter, id_button) {
+    var prompt = gradioApp().getElementById(id);
+    var counter = gradioApp().getElementById(id_counter);
+    var textarea = gradioApp().querySelector(`#${id} > label > textarea`);
+
+    if (opts.disable_token_counters) {
+        counter.style.display = "none";
+        return;
+    }
+
+    if (counter.parentElement == prompt.parentElement) {
+        return;
+    }
+
+    prompt.parentElement.insertBefore(counter, prompt);
+    prompt.parentElement.style.position = "relative";
+
+    promptTokenCountUpdateFunctions[id] = function() {
+        update_token_counter(id_button);
+    };
+    textarea.addEventListener("input", promptTokenCountUpdateFunctions[id]);
+}
+
+function setupTokenCounters() {
+    setupTokenCounting('txt2img_prompt', 'txt2img_token_counter', 'txt2img_token_button');
+    setupTokenCounting('txt2img_neg_prompt', 'txt2img_negative_token_counter', 'txt2img_negative_token_button');
+    setupTokenCounting('img2img_prompt', 'img2img_token_counter', 'img2img_token_button');
+    setupTokenCounting('img2img_neg_prompt', 'img2img_negative_token_counter', 'img2img_negative_token_button');
+}

javascript/ui.js

@@ -1,207 +1,265 @@
 // various functions for interaction with ui.py not large enough to warrant putting them in separate files
 
-function set_theme(theme){
-    gradioURL = window.location.href
+function set_theme(theme) {
+    var gradioURL = window.location.href;
     if (!gradioURL.includes('?__theme=')) {
-      window.location.replace(gradioURL + '?__theme=' + theme);
+        window.location.replace(gradioURL + '?__theme=' + theme);
     }
 }
 
-function selected_gallery_index(){
-    var buttons = gradioApp().querySelectorAll('[style="display: block;"].tabitem div[id$=_gallery] .gallery-item')
-    var button = gradioApp().querySelector('[style="display: block;"].tabitem div[id$=_gallery] .gallery-item.\\!ring-2')
-
-    var result = -1
-    buttons.forEach(function(v, i){ if(v==button) { result = i } })
-
-    return result
+function all_gallery_buttons() {
+    var allGalleryButtons = gradioApp().querySelectorAll('[style="display: block;"].tabitem div[id$=_gallery].gradio-gallery .thumbnails > .thumbnail-item.thumbnail-small');
+    var visibleGalleryButtons = [];
+    allGalleryButtons.forEach(function(elem) {
+        if (elem.parentElement.offsetParent) {
+            visibleGalleryButtons.push(elem);
+        }
+    });
+    return visibleGalleryButtons;
 }
 
-function extract_image_from_gallery(gallery){
-    if(gallery.length == 1){
-        return [gallery[0]]
+function selected_gallery_button() {
+    var allCurrentButtons = gradioApp().querySelectorAll('[style="display: block;"].tabitem div[id$=_gallery].gradio-gallery .thumbnail-item.thumbnail-small.selected');
+    var visibleCurrentButton = null;
+    allCurrentButtons.forEach(function(elem) {
+        if (elem.parentElement.offsetParent) {
+            visibleCurrentButton = elem;
+        }
+    });
+    return visibleCurrentButton;
+}
+
+function selected_gallery_index() {
+    var buttons = all_gallery_buttons();
+    var button = selected_gallery_button();
+
+    var result = -1;
+    buttons.forEach(function(v, i) {
+        if (v == button) {
+            result = i;
+        }
+    });
+
+    return result;
+}
+
+function extract_image_from_gallery(gallery) {
+    if (gallery.length == 0) {
+        return [null];
+    }
+    if (gallery.length == 1) {
+        return [gallery[0]];
     }
 
-    index = selected_gallery_index()
+    var index = selected_gallery_index();
 
-    if (index < 0 || index >= gallery.length){
-        return [null]
+    if (index < 0 || index >= gallery.length) {
+        // Use the first image in the gallery as the default
+        index = 0;
     }
 
     return [gallery[index]];
 }
 
-function args_to_array(args){
-    res = []
-    for(var i=0;i<args.length;i++){
-        res.push(args[i])
-    }
-    return res
-}
+window.args_to_array = Array.from; // Compatibility with e.g. extensions that may expect this to be around
 
-function switch_to_txt2img(){
+function switch_to_txt2img() {
     gradioApp().querySelector('#tabs').querySelectorAll('button')[0].click();
 
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function switch_to_img2img_tab(no){
+function switch_to_img2img_tab(no) {
     gradioApp().querySelector('#tabs').querySelectorAll('button')[1].click();
     gradioApp().getElementById('mode_img2img').querySelectorAll('button')[no].click();
 }
-function switch_to_img2img(){
+function switch_to_img2img() {
     switch_to_img2img_tab(0);
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function switch_to_sketch(){
+function switch_to_sketch() {
     switch_to_img2img_tab(1);
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function switch_to_inpaint(){
+function switch_to_inpaint() {
     switch_to_img2img_tab(2);
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function switch_to_inpaint_sketch(){
+function switch_to_inpaint_sketch() {
     switch_to_img2img_tab(3);
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function switch_to_inpaint(){
-    gradioApp().querySelector('#tabs').querySelectorAll('button')[1].click();
-    gradioApp().getElementById('mode_img2img').querySelectorAll('button')[2].click();
-
-    return args_to_array(arguments);
-}
-
-function switch_to_extras(){
+function switch_to_extras() {
     gradioApp().querySelector('#tabs').querySelectorAll('button')[2].click();
 
-    return args_to_array(arguments);
+    return Array.from(arguments);
 }
 
-function get_tab_index(tabId){
-    var res = 0
-
-    gradioApp().getElementById(tabId).querySelector('div').querySelectorAll('button').forEach(function(button, i){
-        if(button.className.indexOf('bg-white') != -1)
-            res = i
-    })
-
-    return res
-}
-
-function create_tab_index_args(tabId, args){
-    var res = []
-    for(var i=0; i<args.length; i++){
-        res.push(args[i])
+function get_tab_index(tabId) {
+    let buttons = gradioApp().getElementById(tabId).querySelector('div').querySelectorAll('button');
+    for (let i = 0; i < buttons.length; i++) {
+        if (buttons[i].classList.contains('selected')) {
+            return i;
+        }
     }
+    return 0;
+}
 
-    res[0] = get_tab_index(tabId)
-
-    return res
+function create_tab_index_args(tabId, args) {
+    var res = Array.from(args);
+    res[0] = get_tab_index(tabId);
+    return res;
 }
 
 function get_img2img_tab_index() {
-    let res = args_to_array(arguments)
-    res.splice(-2)
-    res[0] = get_tab_index('mode_img2img')
-    return res
+    let res = Array.from(arguments);
+    res.splice(-2);
+    res[0] = get_tab_index('mode_img2img');
+    return res;
 }
 
-function create_submit_args(args){
-    res = []
-    for(var i=0;i<args.length;i++){
-        res.push(args[i])
-    }
+function create_submit_args(args) {
+    var res = Array.from(args);
 
     // As it is currently, txt2img and img2img send back the previous output args (txt2img_gallery, generation_info, html_info) whenever you generate a new image.
     // This can lead to uploading a huge gallery of previously generated images, which leads to an unnecessary delay between submitting and beginning to generate.
     // I don't know why gradio is sending outputs along with inputs, but we can prevent sending the image gallery here, which seems to be an issue for some.
     // If gradio at some point stops sending outputs, this may break something
-    if(Array.isArray(res[res.length - 3])){
-        res[res.length - 3] = null
+    if (Array.isArray(res[res.length - 3])) {
+        res[res.length - 3] = null;
     }
 
-    return res
+    return res;
 }
 
-function showSubmitButtons(tabname, show){
-    gradioApp().getElementById(tabname+'_interrupt').style.display = show ? "none" : "block"
-    gradioApp().getElementById(tabname+'_skip').style.display = show ? "none" : "block"
+function showSubmitButtons(tabname, show) {
+    gradioApp().getElementById(tabname + '_interrupt').style.display = show ? "none" : "block";
+    gradioApp().getElementById(tabname + '_skip').style.display = show ? "none" : "block";
 }
 
-function submit(){
-    rememberGallerySelection('txt2img_gallery')
-    showSubmitButtons('txt2img', false)
+function showRestoreProgressButton(tabname, show) {
+    var button = gradioApp().getElementById(tabname + "_restore_progress");
+    if (!button) return;
 
-    var id = randomId()
-    requestProgress(id, gradioApp().getElementById('txt2img_gallery_container'), gradioApp().getElementById('txt2img_gallery'), function(){
-        showSubmitButtons('txt2img', true)
-
-    })
-
-    var res = create_submit_args(arguments)
-
-    res[0] = id
-
-    return res
+    button.style.display = show ? "flex" : "none";
 }
 
-function submit_img2img(){
-    rememberGallerySelection('img2img_gallery')
-    showSubmitButtons('img2img', false)
+function submit() {
+    showSubmitButtons('txt2img', false);
 
-    var id = randomId()
-    requestProgress(id, gradioApp().getElementById('img2img_gallery_container'), gradioApp().getElementById('img2img_gallery'), function(){
-        showSubmitButtons('img2img', true)
-    })
+    var id = randomId();
+    localStorage.setItem("txt2img_task_id", id);
 
-    var res = create_submit_args(arguments)
+    requestProgress(id, gradioApp().getElementById('txt2img_gallery_container'), gradioApp().getElementById('txt2img_gallery'), function() {
+        showSubmitButtons('txt2img', true);
+        localStorage.removeItem("txt2img_task_id");
+        showRestoreProgressButton('txt2img', false);
+    });
 
-    res[0] = id
-    res[1] = get_tab_index('mode_img2img')
+    var res = create_submit_args(arguments);
 
-    return res
+    res[0] = id;
+
+    return res;
 }
 
-function modelmerger(){
-    var id = randomId()
-    requestProgress(id, gradioApp().getElementById('modelmerger_results_panel'), null, function(){})
+function submit_img2img() {
+    showSubmitButtons('img2img', false);
 
-    var res = create_submit_args(arguments)
-    res[0] = id
-    return res
+    var id = randomId();
+    localStorage.setItem("img2img_task_id", id);
+
+    requestProgress(id, gradioApp().getElementById('img2img_gallery_container'), gradioApp().getElementById('img2img_gallery'), function() {
+        showSubmitButtons('img2img', true);
+        localStorage.removeItem("img2img_task_id");
+        showRestoreProgressButton('img2img', false);
+    });
+
+    var res = create_submit_args(arguments);
+
+    res[0] = id;
+    res[1] = get_tab_index('mode_img2img');
+
+    return res;
+}
+
+function restoreProgressTxt2img() {
+    showRestoreProgressButton("txt2img", false);
+    var id = localStorage.getItem("txt2img_task_id");
+
+    id = localStorage.getItem("txt2img_task_id");
+
+    if (id) {
+        requestProgress(id, gradioApp().getElementById('txt2img_gallery_container'), gradioApp().getElementById('txt2img_gallery'), function() {
+            showSubmitButtons('txt2img', true);
+        }, null, 0);
+    }
+
+    return id;
+}
+
+function restoreProgressImg2img() {
+    showRestoreProgressButton("img2img", false);
+
+    var id = localStorage.getItem("img2img_task_id");
+
+    if (id) {
+        requestProgress(id, gradioApp().getElementById('img2img_gallery_container'), gradioApp().getElementById('img2img_gallery'), function() {
+            showSubmitButtons('img2img', true);
+        }, null, 0);
+    }
+
+    return id;
+}
+
+
+onUiLoaded(function() {
+    showRestoreProgressButton('txt2img', localStorage.getItem("txt2img_task_id"));
+    showRestoreProgressButton('img2img', localStorage.getItem("img2img_task_id"));
+});
+
+
+function modelmerger() {
+    var id = randomId();
+    requestProgress(id, gradioApp().getElementById('modelmerger_results_panel'), null, function() {});
+
+    var res = create_submit_args(arguments);
+    res[0] = id;
+    return res;
 }
 
 
 function ask_for_style_name(_, prompt_text, negative_prompt_text) {
-    name_ = prompt('Style name:')
-    return [name_, prompt_text, negative_prompt_text]
+    var name_ = prompt('Style name:');
+    return [name_, prompt_text, negative_prompt_text];
 }
 
 function confirm_clear_prompt(prompt, negative_prompt) {
-    if(confirm("Delete prompt?")) {
-        prompt = ""
-        negative_prompt = ""
+    if (confirm("Delete prompt?")) {
+        prompt = "";
+        negative_prompt = "";
     }
 
-    return [prompt, negative_prompt]
+    return [prompt, negative_prompt];
 }
 
-opts = {}
-onUiUpdate(function(){
-	if(Object.keys(opts).length != 0) return;
 
-	json_elem = gradioApp().getElementById('settings_json')
-	if(json_elem == null) return;
+var opts = {};
+onAfterUiUpdate(function() {
+    if (Object.keys(opts).length != 0) return;
 
-    var textarea = json_elem.querySelector('textarea')
-    var jsdata = textarea.value
-    opts = JSON.parse(jsdata)
-    executeCallbacks(optionsChangedCallbacks);
+    var json_elem = gradioApp().getElementById('settings_json');
+    if (json_elem == null) return;
+
+    var textarea = json_elem.querySelector('textarea');
+    var jsdata = textarea.value;
+    opts = JSON.parse(jsdata);
+
+    executeCallbacks(optionsChangedCallbacks); /*global optionsChangedCallbacks*/
 
     Object.defineProperty(textarea, 'value', {
         set: function(newValue) {
@@ -210,7 +268,7 @@ onUiUpdate(function(){
             valueProp.set.call(textarea, newValue);
 
             if (oldValue != newValue) {
-                opts = JSON.parse(textarea.value)
+                opts = JSON.parse(textarea.value);
             }
 
             executeCallbacks(optionsChangedCallbacks);
@@ -221,91 +279,109 @@ onUiUpdate(function(){
         }
     });
 
-    json_elem.parentElement.style.display="none"
+    json_elem.parentElement.style.display = "none";
 
-    function registerTextarea(id, id_counter, id_button){
-        var prompt = gradioApp().getElementById(id)
-        var counter = gradioApp().getElementById(id_counter)
-        var textarea = gradioApp().querySelector("#" + id + " > label > textarea");
+    setupTokenCounters();
 
-        if(counter.parentElement == prompt.parentElement){
-            return
-        }
+    var show_all_pages = gradioApp().getElementById('settings_show_all_pages');
+    var settings_tabs = gradioApp().querySelector('#settings div');
+    if (show_all_pages && settings_tabs) {
+        settings_tabs.appendChild(show_all_pages);
+        show_all_pages.onclick = function() {
+            gradioApp().querySelectorAll('#settings > div').forEach(function(elem) {
+                if (elem.id == "settings_tab_licenses") {
+                    return;
+                }
 
-
-        prompt.parentElement.insertBefore(counter, prompt)
-        counter.classList.add("token-counter")
-        prompt.parentElement.style.position = "relative"
-
-		textarea.addEventListener("input", function(){
-		    update_token_counter(id_button);
-		});
-    }
-
-    registerTextarea('txt2img_prompt', 'txt2img_token_counter', 'txt2img_token_button')
-    registerTextarea('txt2img_neg_prompt', 'txt2img_negative_token_counter', 'txt2img_negative_token_button')
-    registerTextarea('img2img_prompt', 'img2img_token_counter', 'img2img_token_button')
-    registerTextarea('img2img_neg_prompt', 'img2img_negative_token_counter', 'img2img_negative_token_button')
-
-    show_all_pages = gradioApp().getElementById('settings_show_all_pages')
-    settings_tabs = gradioApp().querySelector('#settings div')
-    if(show_all_pages && settings_tabs){
-        settings_tabs.appendChild(show_all_pages)
-        show_all_pages.onclick = function(){
-            gradioApp().querySelectorAll('#settings > div').forEach(function(elem){
                 elem.style.display = "block";
-            })
-        }
+            });
+        };
     }
-})
+});
 
-onOptionsChanged(function(){
-    elem = gradioApp().getElementById('sd_checkpoint_hash')
-    sd_checkpoint_hash = opts.sd_checkpoint_hash || ""
-    shorthash = sd_checkpoint_hash.substr(0,10)
+onOptionsChanged(function() {
+    var elem = gradioApp().getElementById('sd_checkpoint_hash');
+    var sd_checkpoint_hash = opts.sd_checkpoint_hash || "";
+    var shorthash = sd_checkpoint_hash.substring(0, 10);
 
-	if(elem && elem.textContent != shorthash){
-	    elem.textContent = shorthash
-	    elem.title = sd_checkpoint_hash
-	    elem.href = "https://google.com/search?q=" + sd_checkpoint_hash
-	}
-})
+    if (elem && elem.textContent != shorthash) {
+        elem.textContent = shorthash;
+        elem.title = sd_checkpoint_hash;
+        elem.href = "https://google.com/search?q=" + sd_checkpoint_hash;
+    }
+});
 
 let txt2img_textarea, img2img_textarea = undefined;
-let wait_time = 800
-let token_timeout;
 
-function update_txt2img_tokens(...args) {
-	update_token_counter("txt2img_token_button")
-	if (args.length == 2)
-		return args[0]
-	return args;
-}
+function restart_reload() {
+    document.body.innerHTML = '<h1 style="font-family:monospace;margin-top:20%;color:lightgray;text-align:center;">Reloading...</h1>';
 
-function update_img2img_tokens(...args) {
-	update_token_counter("img2img_token_button")
-	if (args.length == 2)
-		return args[0]
-	return args;
-}
+    var requestPing = function() {
+        requestGet("./internal/ping", {}, function(data) {
+            location.reload();
+        }, function() {
+            setTimeout(requestPing, 500);
+        });
+    };
 
-function update_token_counter(button_id) {
-	if (token_timeout)
-		clearTimeout(token_timeout);
-	token_timeout = setTimeout(() => gradioApp().getElementById(button_id)?.click(), wait_time);
-}
+    setTimeout(requestPing, 2000);
 
-function restart_reload(){
-    document.body.innerHTML='<h1 style="font-family:monospace;margin-top:20%;color:lightgray;text-align:center;">Reloading...</h1>';
-    setTimeout(function(){location.reload()},2000)
-
-    return []
+    return [];
 }
 
 // Simulate an `input` DOM event for Gradio Textbox component. Needed after you edit its contents in javascript, otherwise your edits
 // will only visible on web page and not sent to python.
-function updateInput(target){
-	let e = new Event("input", { bubbles: true })
-	Object.defineProperty(e, "target", {value: target})
-	target.dispatchEvent(e);
+function updateInput(target) {
+    let e = new Event("input", {bubbles: true});
+    Object.defineProperty(e, "target", {value: target});
+    target.dispatchEvent(e);
+}
+
+
+var desiredCheckpointName = null;
+function selectCheckpoint(name) {
+    desiredCheckpointName = name;
+    gradioApp().getElementById('change_checkpoint').click();
+}
+
+function currentImg2imgSourceResolution(w, h, scaleBy) {
+    var img = gradioApp().querySelector('#mode_img2img > div[style="display: block;"] img');
+    return img ? [img.naturalWidth, img.naturalHeight, scaleBy] : [0, 0, scaleBy];
+}
+
+function updateImg2imgResizeToTextAfterChangingImage() {
+    // At the time this is called from gradio, the image has no yet been replaced.
+    // There may be a better solution, but this is simple and straightforward so I'm going with it.
+
+    setTimeout(function() {
+        gradioApp().getElementById('img2img_update_resize_to').click();
+    }, 500);
+
+    return [];
+
+}
+
+
+
+function setRandomSeed(elem_id) {
+    var input = gradioApp().querySelector("#" + elem_id + " input");
+    if (!input) return [];
+
+    input.value = "-1";
+    updateInput(input);
+    return [];
+}
+
+function switchWidthHeight(tabname) {
+    var width = gradioApp().querySelector("#" + tabname + "_width input[type=number]");
+    var height = gradioApp().querySelector("#" + tabname + "_height input[type=number]");
+    if (!width || !height) return [];
+
+    var tmp = width.value;
+    width.value = height.value;
+    height.value = tmp;
+
+    updateInput(width);
+    updateInput(height);
+    return [];
 }

javascript/ui_settings_hints.js

@@ -0,0 +1,62 @@
+// various hints and extra info for the settings tab
+
+var settingsHintsSetup = false;
+
+onOptionsChanged(function() {
+    if (settingsHintsSetup) return;
+    settingsHintsSetup = true;
+
+    gradioApp().querySelectorAll('#settings [id^=setting_]').forEach(function(div) {
+        var name = div.id.substr(8);
+        var commentBefore = opts._comments_before[name];
+        var commentAfter = opts._comments_after[name];
+
+        if (!commentBefore && !commentAfter) return;
+
+        var span = null;
+        if (div.classList.contains('gradio-checkbox')) span = div.querySelector('label span');
+        else if (div.classList.contains('gradio-checkboxgroup')) span = div.querySelector('span').firstChild;
+        else if (div.classList.contains('gradio-radio')) span = div.querySelector('span').firstChild;
+        else span = div.querySelector('label span').firstChild;
+
+        if (!span) return;
+
+        if (commentBefore) {
+            var comment = document.createElement('DIV');
+            comment.className = 'settings-comment';
+            comment.innerHTML = commentBefore;
+            span.parentElement.insertBefore(document.createTextNode('\xa0'), span);
+            span.parentElement.insertBefore(comment, span);
+            span.parentElement.insertBefore(document.createTextNode('\xa0'), span);
+        }
+        if (commentAfter) {
+            comment = document.createElement('DIV');
+            comment.className = 'settings-comment';
+            comment.innerHTML = commentAfter;
+            span.parentElement.insertBefore(comment, span.nextSibling);
+            span.parentElement.insertBefore(document.createTextNode('\xa0'), span.nextSibling);
+        }
+    });
+});
+
+function settingsHintsShowQuicksettings() {
+    requestGet("./internal/quicksettings-hint", {}, function(data) {
+        var table = document.createElement('table');
+        table.className = 'popup-table';
+
+        data.forEach(function(obj) {
+            var tr = document.createElement('tr');
+            var td = document.createElement('td');
+            td.textContent = obj.name;
+            tr.appendChild(td);
+
+            td = document.createElement('td');
+            td.textContent = obj.label;
+            tr.appendChild(td);
+
+            table.appendChild(tr);
+        });
+
+        popup(table);
+    });
+}

launch.py

@@ -1,330 +1,38 @@
-# this scripts installs necessary requirements and launches main program in webui.py
-import subprocess
-import os
-import sys
-import importlib.util
-import shlex
-import platform
-import argparse
-import json
+from modules import launch_utils
 
-dir_repos = "repositories"
-dir_extensions = "extensions"
-python = sys.executable
-git = os.environ.get('GIT', "git")
-index_url = os.environ.get('INDEX_URL', "")
-stored_commit_hash = None
-skip_install = False
 
+args = launch_utils.args
+python = launch_utils.python
+git = launch_utils.git
+index_url = launch_utils.index_url
+dir_repos = launch_utils.dir_repos
 
-def commit_hash():
-    global stored_commit_hash
+commit_hash = launch_utils.commit_hash
+git_tag = launch_utils.git_tag
 
-    if stored_commit_hash is not None:
-        return stored_commit_hash
+run = launch_utils.run
+is_installed = launch_utils.is_installed
+repo_dir = launch_utils.repo_dir
 
-    try:
-        stored_commit_hash = run(f"{git} rev-parse HEAD").strip()
-    except Exception:
-        stored_commit_hash = "<none>"
+run_pip = launch_utils.run_pip
+check_run_python = launch_utils.check_run_python
+git_clone = launch_utils.git_clone
+git_pull_recursive = launch_utils.git_pull_recursive
+run_extension_installer = launch_utils.run_extension_installer
+prepare_environment = launch_utils.prepare_environment
+configure_for_tests = launch_utils.configure_for_tests
+start = launch_utils.start
 
-    return stored_commit_hash
 
+def main():
+    if not args.skip_prepare_environment:
+        prepare_environment()
 
-def extract_arg(args, name):
-    return [x for x in args if x != name], name in args
+    if args.test_server:
+        configure_for_tests()
 
-
-def extract_opt(args, name):
-    opt = None
-    is_present = False
-    if name in args:
-        is_present = True
-        idx = args.index(name)
-        del args[idx]
-        if idx < len(args) and args[idx][0] != "-":
-            opt = args[idx]
-            del args[idx]
-    return args, is_present, opt
-
-
-def run(command, desc=None, errdesc=None, custom_env=None, live=False):
-    if desc is not None:
-        print(desc)
-
-    if live:
-        result = subprocess.run(command, shell=True, env=os.environ if custom_env is None else custom_env)
-        if result.returncode != 0:
-            raise RuntimeError(f"""{errdesc or 'Error running command'}.
-Command: {command}
-Error code: {result.returncode}""")
-
-        return ""
-
-    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ if custom_env is None else custom_env)
-
-    if result.returncode != 0:
-
-        message = f"""{errdesc or 'Error running command'}.
-Command: {command}
-Error code: {result.returncode}
-stdout: {result.stdout.decode(encoding="utf8", errors="ignore") if len(result.stdout)>0 else '<empty>'}
-stderr: {result.stderr.decode(encoding="utf8", errors="ignore") if len(result.stderr)>0 else '<empty>'}
-"""
-        raise RuntimeError(message)
-
-    return result.stdout.decode(encoding="utf8", errors="ignore")
-
-
-def check_run(command):
-    result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-    return result.returncode == 0
-
-
-def is_installed(package):
-    try:
-        spec = importlib.util.find_spec(package)
-    except ModuleNotFoundError:
-        return False
-
-    return spec is not None
-
-
-def repo_dir(name):
-    return os.path.join(dir_repos, name)
-
-
-def run_python(code, desc=None, errdesc=None):
-    return run(f'"{python}" -c "{code}"', desc, errdesc)
-
-
-def run_pip(args, desc=None):
-    if skip_install:
-        return
-
-    index_url_line = f' --index-url {index_url}' if index_url != '' else ''
-    return run(f'"{python}" -m pip {args} --prefer-binary{index_url_line}', desc=f"Installing {desc}", errdesc=f"Couldn't install {desc}")
-
-
-def check_run_python(code):
-    return check_run(f'"{python}" -c "{code}"')
-
-
-def git_clone(url, dir, name, commithash=None):
-    # TODO clone into temporary dir and move if successful
-
-    if os.path.exists(dir):
-        if commithash is None:
-            return
-
-        current_hash = run(f'"{git}" -C "{dir}" rev-parse HEAD', None, f"Couldn't determine {name}'s hash: {commithash}").strip()
-        if current_hash == commithash:
-            return
-
-        run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}")
-        run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}")
-        return
-
-    run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}")
-
-    if commithash is not None:
-        run(f'"{git}" -C "{dir}" checkout {commithash}', None, "Couldn't checkout {name}'s hash: {commithash}")
-
-        
-def version_check(commit):
-    try:
-        import requests
-        commits = requests.get('https://api.github.com/repos/AUTOMATIC1111/stable-diffusion-webui/branches/master').json()
-        if commit != "<none>" and commits['commit']['sha'] != commit:
-            print("--------------------------------------------------------")
-            print("| You are not up to date with the most recent release. |")
-            print("| Consider running `git pull` to update.               |")
-            print("--------------------------------------------------------")
-        elif commits['commit']['sha'] == commit:
-            print("You are up to date with the most recent release.")
-        else:
-            print("Not a git clone, can't perform version check.")
-    except Exception as e:
-        print("version check failed", e)
-
-
-def run_extension_installer(extension_dir):
-    path_installer = os.path.join(extension_dir, "install.py")
-    if not os.path.isfile(path_installer):
-        return
-
-    try:
-        env = os.environ.copy()
-        env['PYTHONPATH'] = os.path.abspath(".")
-
-        print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env))
-    except Exception as e:
-        print(e, file=sys.stderr)
-
-
-def list_extensions(settings_file):
-    settings = {}
-
-    try:
-        if os.path.isfile(settings_file):
-            with open(settings_file, "r", encoding="utf8") as file:
-                settings = json.load(file)
-    except Exception as e:
-        print(e, file=sys.stderr)
-
-    disabled_extensions = set(settings.get('disabled_extensions', []))
-
-    return [x for x in os.listdir(dir_extensions) if x not in disabled_extensions]
-
-
-def run_extensions_installers(settings_file):
-    if not os.path.isdir(dir_extensions):
-        return
-
-    for dirname_extension in list_extensions(settings_file):
-        run_extension_installer(os.path.join(dir_extensions, dirname_extension))
-
-
-def prepare_environment():
-    global skip_install
-
-    pip_installer_location = os.environ.get('PIP_INSTALLER_LOCATION', None)
-
-    torch_command = os.environ.get('TORCH_COMMAND', "pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 --extra-index-url https://download.pytorch.org/whl/cu117")
-    requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt")
-    commandline_args = os.environ.get('COMMANDLINE_ARGS', "")
-
-    gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "git+https://github.com/TencentARC/GFPGAN.git@8d2447a2d918f8eba5a4a01463fd48e45126a379")
-    clip_package = os.environ.get('CLIP_PACKAGE', "git+https://github.com/openai/CLIP.git@d50d76daa670286dd6cacf3bcd80b5e4823fc8e1")
-    openclip_package = os.environ.get('OPENCLIP_PACKAGE', "git+https://github.com/mlfoundations/open_clip.git@bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b")
-
-    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
-    taming_transformers_repo = os.environ.get('TAMING_TRANSFORMERS_REPO', "https://github.com/CompVis/taming-transformers.git")
-    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
-    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
-    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')
-
-    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "47b6b607fdd31875c9279cd2f4f16b92e4ea958e")
-    taming_transformers_commit_hash = os.environ.get('TAMING_TRANSFORMERS_COMMIT_HASH', "24268930bf1dce879235a7fddd0b2355b84d7ea6")
-    k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "5b3af030dd83e0297272d861c19477735d0317ec")
-    codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
-    blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9")
-
-    sys.argv += shlex.split(commandline_args)
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--ui-settings-file", type=str, help="filename to use for ui settings", default='config.json')
-    args, _ = parser.parse_known_args(sys.argv)
-
-    sys.argv, _ = extract_arg(sys.argv, '-f')
-    sys.argv, skip_torch_cuda_test = extract_arg(sys.argv, '--skip-torch-cuda-test')
-    sys.argv, reinstall_xformers = extract_arg(sys.argv, '--reinstall-xformers')
-    sys.argv, reinstall_torch = extract_arg(sys.argv, '--reinstall-torch')
-    sys.argv, update_check = extract_arg(sys.argv, '--update-check')
-    sys.argv, run_tests, test_dir = extract_opt(sys.argv, '--tests')
-    sys.argv, skip_install = extract_arg(sys.argv, '--skip-install')
-    xformers = '--xformers' in sys.argv
-    ngrok = '--ngrok' in sys.argv
-
-    commit = commit_hash()
-
-    print(f"Python {sys.version}")
-    print(f"Commit hash: {commit}")
-
-    if pip_installer_location is not None and not is_installed("pip"):
-        run(f'"{python}" "{pip_installer_location}"', "Installing pip", "Couldn't install pip")
-
-    if reinstall_torch or not is_installed("torch") or not is_installed("torchvision"):
-        run(f'"{python}" -m {torch_command}', "Installing torch and torchvision", "Couldn't install torch", live=True)
-
-    if not skip_torch_cuda_test:
-        run_python("import torch; assert torch.cuda.is_available(), 'Torch is not able to use GPU; add --skip-torch-cuda-test to COMMANDLINE_ARGS variable to disable this check'")
-
-    if not is_installed("gfpgan"):
-        run_pip(f"install {gfpgan_package}", "gfpgan")
-
-    if not is_installed("clip"):
-        run_pip(f"install {clip_package}", "clip")
-
-    if not is_installed("open_clip"):
-        run_pip(f"install {openclip_package}", "open_clip")
-
-    if (not is_installed("xformers") or reinstall_xformers) and xformers:
-        if platform.system() == "Windows":
-            if platform.python_version().startswith("3.10"):
-                run_pip(f"install -U -I --no-deps xformers==0.0.16rc425", "xformers")
-            else:
-                print("Installation of xformers is not supported in this version of Python.")
-                print("You can also check this and build manually: https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Xformers#building-xformers-on-windows-by-duckness")
-                if not is_installed("xformers"):
-                    exit(0)
-        elif platform.system() == "Linux":
-            run_pip("install xformers==0.0.16rc425", "xformers")
-
-    if not is_installed("pyngrok") and ngrok:
-        run_pip("install pyngrok", "ngrok")
-
-    os.makedirs(dir_repos, exist_ok=True)
-
-    git_clone(stable_diffusion_repo, repo_dir('stable-diffusion-stability-ai'), "Stable Diffusion", stable_diffusion_commit_hash)
-    git_clone(taming_transformers_repo, repo_dir('taming-transformers'), "Taming Transformers", taming_transformers_commit_hash)
-    git_clone(k_diffusion_repo, repo_dir('k-diffusion'), "K-diffusion", k_diffusion_commit_hash)
-    git_clone(codeformer_repo, repo_dir('CodeFormer'), "CodeFormer", codeformer_commit_hash)
-    git_clone(blip_repo, repo_dir('BLIP'), "BLIP", blip_commit_hash)
-
-    if not is_installed("lpips"):
-        run_pip(f"install -r {os.path.join(repo_dir('CodeFormer'), 'requirements.txt')}", "requirements for CodeFormer")
-
-    run_pip(f"install -r {requirements_file}", "requirements for Web UI")
-
-    run_extensions_installers(settings_file=args.ui_settings_file)
-
-    if update_check:
-        version_check(commit)
-    
-    if "--exit" in sys.argv:
-        print("Exiting because of --exit argument")
-        exit(0)
-
-    if run_tests:
-        exitcode = tests(test_dir)
-        exit(exitcode)
-
-
-def tests(test_dir):
-    if "--api" not in sys.argv:
-        sys.argv.append("--api")
-    if "--ckpt" not in sys.argv:
-        sys.argv.append("--ckpt")
-        sys.argv.append("./test/test_files/empty.pt")
-    if "--skip-torch-cuda-test" not in sys.argv:
-        sys.argv.append("--skip-torch-cuda-test")
-    if "--disable-nan-check" not in sys.argv:
-        sys.argv.append("--disable-nan-check")
-
-    print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}")
-
-    os.environ['COMMANDLINE_ARGS'] = ""
-    with open('test/stdout.txt', "w", encoding="utf8") as stdout, open('test/stderr.txt', "w", encoding="utf8") as stderr:
-        proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr)
-
-    import test.server_poll
-    exitcode = test.server_poll.run_tests(proc, test_dir)
-
-    print(f"Stopping Web UI process with id {proc.pid}")
-    proc.kill()
-    return exitcode
-
-
-def start():
-    print(f"Launching {'API server' if '--nowebui' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}")
-    import webui
-    if '--nowebui' in sys.argv:
-        webui.api_only()
-    else:
-        webui.webui()
+    start()
 
 
 if __name__ == "__main__":
-    prepare_environment()
-    start()
+    main()

modules/api/api.py

@@ -3,39 +3,48 @@ import io
 import time
 import datetime
 import uvicorn
+import gradio as gr
 from threading import Lock
 from io import BytesIO
-from gradio.processing_utils import decode_base64_to_file
-from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, Response
+from fastapi import APIRouter, Depends, FastAPI, Request, Response
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
+from fastapi.exceptions import HTTPException
+from fastapi.responses import JSONResponse
+from fastapi.encoders import jsonable_encoder
 from secrets import compare_digest
 
 import modules.shared as shared
-from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing
-from modules.api.models import *
+from modules import sd_samplers, deepbooru, sd_hijack, images, scripts, ui, postprocessing, errors
+from modules.api import models
+from modules.shared import opts
 from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img, process_images
 from modules.textual_inversion.textual_inversion import create_embedding, train_embedding
 from modules.textual_inversion.preprocess import preprocess
 from modules.hypernetworks.hypernetwork import create_hypernetwork, train_hypernetwork
 from PIL import PngImagePlugin,Image
-from modules.sd_models import checkpoints_list, find_checkpoint_config
+from modules.sd_models import checkpoints_list, unload_model_weights, reload_model_weights
+from modules.sd_vae import vae_dict
+from modules.sd_models_config import find_checkpoint_config_near_filename
 from modules.realesrgan_model import get_realesrgan_models
 from modules import devices
-from typing import List
+from typing import Dict, List, Any
 import piexif
 import piexif.helper
 
+
 def upscaler_to_index(name: str):
     try:
         return [x.name.lower() for x in shared.sd_upscalers].index(name.lower())
-    except:
-        raise HTTPException(status_code=400, detail=f"Invalid upscaler, needs to be one of these: {' , '.join([x.name for x in sd_upscalers])}")
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Invalid upscaler, needs to be one of these: {' , '.join([x.name for x in shared.sd_upscalers])}") from e
+
 
 def script_name_to_index(name, scripts):
     try:
         return [script.title().lower() for script in scripts].index(name.lower())
-    except:
-        raise HTTPException(status_code=422, detail=f"Script '{name}' not found")
+    except Exception as e:
+        raise HTTPException(status_code=422, detail=f"Script '{name}' not found") from e
+
 
 def validate_sampler_name(name):
     config = sd_samplers.all_samplers_map.get(name, None)
@@ -44,20 +53,23 @@ def validate_sampler_name(name):
 
     return name
 
+
 def setUpscalers(req: dict):
     reqDict = vars(req)
     reqDict['extras_upscaler_1'] = reqDict.pop('upscaler_1', None)
     reqDict['extras_upscaler_2'] = reqDict.pop('upscaler_2', None)
     return reqDict
 
+
 def decode_base64_to_image(encoding):
     if encoding.startswith("data:image/"):
         encoding = encoding.split(";")[1].split(",")[1]
     try:
         image = Image.open(BytesIO(base64.b64decode(encoding)))
         return image
-    except Exception as err:
-        raise HTTPException(status_code=500, detail="Invalid encoded image")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="Invalid encoded image") from e
+
 
 def encode_pil_to_base64(image):
     with io.BytesIO() as output_bytes:
@@ -88,7 +100,17 @@ def encode_pil_to_base64(image):
 
     return base64.b64encode(bytes_data)
 
+
 def api_middleware(app: FastAPI):
+    rich_available = True
+    try:
+        import anyio # importing just so it can be placed on silent list
+        import starlette # importing just so it can be placed on silent list
+        from rich.console import Console
+        console = Console()
+    except Exception:
+        rich_available = False
+
     @app.middleware("http")
     async def log_and_time(req: Request, call_next):
         ts = time.time()
@@ -109,11 +131,42 @@ def api_middleware(app: FastAPI):
             ))
         return res
 
+    def handle_exception(request: Request, e: Exception):
+        err = {
+            "error": type(e).__name__,
+            "detail": vars(e).get('detail', ''),
+            "body": vars(e).get('body', ''),
+            "errors": str(e),
+        }
+        if not isinstance(e, HTTPException): # do not print backtrace on known httpexceptions
+            message = f"API error: {request.method}: {request.url} {err}"
+            if rich_available:
+                print(message)
+                console.print_exception(show_locals=True, max_frames=2, extra_lines=1, suppress=[anyio, starlette], word_wrap=False, width=min([console.width, 200]))
+            else:
+                errors.report(message, exc_info=True)
+        return JSONResponse(status_code=vars(e).get('status_code', 500), content=jsonable_encoder(err))
+
+    @app.middleware("http")
+    async def exception_handling(request: Request, call_next):
+        try:
+            return await call_next(request)
+        except Exception as e:
+            return handle_exception(request, e)
+
+    @app.exception_handler(Exception)
+    async def fastapi_exception_handler(request: Request, e: Exception):
+        return handle_exception(request, e)
+
+    @app.exception_handler(HTTPException)
+    async def http_exception_handler(request: Request, e: HTTPException):
+        return handle_exception(request, e)
+
 
 class Api:
     def __init__(self, app: FastAPI, queue_lock: Lock):
         if shared.cmd_opts.api_auth:
-            self.credentials = dict()
+            self.credentials = {}
             for auth in shared.cmd_opts.api_auth.split(","):
                 user, password = auth.split(":")
                 self.credentials[user] = password
@@ -122,33 +175,42 @@ class Api:
         self.app = app
         self.queue_lock = queue_lock
         api_middleware(self.app)
-        self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=TextToImageResponse)
-        self.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=ImageToImageResponse)
-        self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=ExtrasSingleImageResponse)
-        self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=ExtrasBatchImagesResponse)
-        self.add_api_route("/sdapi/v1/png-info", self.pnginfoapi, methods=["POST"], response_model=PNGInfoResponse)
-        self.add_api_route("/sdapi/v1/progress", self.progressapi, methods=["GET"], response_model=ProgressResponse)
+        self.add_api_route("/sdapi/v1/txt2img", self.text2imgapi, methods=["POST"], response_model=models.TextToImageResponse)
+        self.add_api_route("/sdapi/v1/img2img", self.img2imgapi, methods=["POST"], response_model=models.ImageToImageResponse)
+        self.add_api_route("/sdapi/v1/extra-single-image", self.extras_single_image_api, methods=["POST"], response_model=models.ExtrasSingleImageResponse)
+        self.add_api_route("/sdapi/v1/extra-batch-images", self.extras_batch_images_api, methods=["POST"], response_model=models.ExtrasBatchImagesResponse)
+        self.add_api_route("/sdapi/v1/png-info", self.pnginfoapi, methods=["POST"], response_model=models.PNGInfoResponse)
+        self.add_api_route("/sdapi/v1/progress", self.progressapi, methods=["GET"], response_model=models.ProgressResponse)
         self.add_api_route("/sdapi/v1/interrogate", self.interrogateapi, methods=["POST"])
         self.add_api_route("/sdapi/v1/interrupt", self.interruptapi, methods=["POST"])
         self.add_api_route("/sdapi/v1/skip", self.skip, methods=["POST"])
-        self.add_api_route("/sdapi/v1/options", self.get_config, methods=["GET"], response_model=OptionsModel)
+        self.add_api_route("/sdapi/v1/options", self.get_config, methods=["GET"], response_model=models.OptionsModel)
         self.add_api_route("/sdapi/v1/options", self.set_config, methods=["POST"])
-        self.add_api_route("/sdapi/v1/cmd-flags", self.get_cmd_flags, methods=["GET"], response_model=FlagsModel)
-        self.add_api_route("/sdapi/v1/samplers", self.get_samplers, methods=["GET"], response_model=List[SamplerItem])
-        self.add_api_route("/sdapi/v1/upscalers", self.get_upscalers, methods=["GET"], response_model=List[UpscalerItem])
-        self.add_api_route("/sdapi/v1/sd-models", self.get_sd_models, methods=["GET"], response_model=List[SDModelItem])
-        self.add_api_route("/sdapi/v1/hypernetworks", self.get_hypernetworks, methods=["GET"], response_model=List[HypernetworkItem])
-        self.add_api_route("/sdapi/v1/face-restorers", self.get_face_restorers, methods=["GET"], response_model=List[FaceRestorerItem])
-        self.add_api_route("/sdapi/v1/realesrgan-models", self.get_realesrgan_models, methods=["GET"], response_model=List[RealesrganItem])
-        self.add_api_route("/sdapi/v1/prompt-styles", self.get_prompt_styles, methods=["GET"], response_model=List[PromptStyleItem])
-        self.add_api_route("/sdapi/v1/embeddings", self.get_embeddings, methods=["GET"], response_model=EmbeddingsResponse)
+        self.add_api_route("/sdapi/v1/cmd-flags", self.get_cmd_flags, methods=["GET"], response_model=models.FlagsModel)
+        self.add_api_route("/sdapi/v1/samplers", self.get_samplers, methods=["GET"], response_model=List[models.SamplerItem])
+        self.add_api_route("/sdapi/v1/upscalers", self.get_upscalers, methods=["GET"], response_model=List[models.UpscalerItem])
+        self.add_api_route("/sdapi/v1/latent-upscale-modes", self.get_latent_upscale_modes, methods=["GET"], response_model=List[models.LatentUpscalerModeItem])
+        self.add_api_route("/sdapi/v1/sd-models", self.get_sd_models, methods=["GET"], response_model=List[models.SDModelItem])
+        self.add_api_route("/sdapi/v1/sd-vae", self.get_sd_vaes, methods=["GET"], response_model=List[models.SDVaeItem])
+        self.add_api_route("/sdapi/v1/hypernetworks", self.get_hypernetworks, methods=["GET"], response_model=List[models.HypernetworkItem])
+        self.add_api_route("/sdapi/v1/face-restorers", self.get_face_restorers, methods=["GET"], response_model=List[models.FaceRestorerItem])
+        self.add_api_route("/sdapi/v1/realesrgan-models", self.get_realesrgan_models, methods=["GET"], response_model=List[models.RealesrganItem])
+        self.add_api_route("/sdapi/v1/prompt-styles", self.get_prompt_styles, methods=["GET"], response_model=List[models.PromptStyleItem])
+        self.add_api_route("/sdapi/v1/embeddings", self.get_embeddings, methods=["GET"], response_model=models.EmbeddingsResponse)
         self.add_api_route("/sdapi/v1/refresh-checkpoints", self.refresh_checkpoints, methods=["POST"])
-        self.add_api_route("/sdapi/v1/create/embedding", self.create_embedding, methods=["POST"], response_model=CreateResponse)
-        self.add_api_route("/sdapi/v1/create/hypernetwork", self.create_hypernetwork, methods=["POST"], response_model=CreateResponse)
-        self.add_api_route("/sdapi/v1/preprocess", self.preprocess, methods=["POST"], response_model=PreprocessResponse)
-        self.add_api_route("/sdapi/v1/train/embedding", self.train_embedding, methods=["POST"], response_model=TrainResponse)
-        self.add_api_route("/sdapi/v1/train/hypernetwork", self.train_hypernetwork, methods=["POST"], response_model=TrainResponse)
-        self.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=MemoryResponse)
+        self.add_api_route("/sdapi/v1/create/embedding", self.create_embedding, methods=["POST"], response_model=models.CreateResponse)
+        self.add_api_route("/sdapi/v1/create/hypernetwork", self.create_hypernetwork, methods=["POST"], response_model=models.CreateResponse)
+        self.add_api_route("/sdapi/v1/preprocess", self.preprocess, methods=["POST"], response_model=models.PreprocessResponse)
+        self.add_api_route("/sdapi/v1/train/embedding", self.train_embedding, methods=["POST"], response_model=models.TrainResponse)
+        self.add_api_route("/sdapi/v1/train/hypernetwork", self.train_hypernetwork, methods=["POST"], response_model=models.TrainResponse)
+        self.add_api_route("/sdapi/v1/memory", self.get_memory, methods=["GET"], response_model=models.MemoryResponse)
+        self.add_api_route("/sdapi/v1/unload-checkpoint", self.unloadapi, methods=["POST"])
+        self.add_api_route("/sdapi/v1/reload-checkpoint", self.reloadapi, methods=["POST"])
+        self.add_api_route("/sdapi/v1/scripts", self.get_scripts_list, methods=["GET"], response_model=models.ScriptsList)
+        self.add_api_route("/sdapi/v1/script-info", self.get_script_info, methods=["GET"], response_model=List[models.ScriptInfo])
+
+        self.default_script_arg_txt2img = []
+        self.default_script_arg_img2img = []
 
     def add_api_route(self, path: str, endpoint, **kwargs):
         if shared.cmd_opts.api_auth:
@@ -162,98 +224,186 @@ class Api:
 
         raise HTTPException(status_code=401, detail="Incorrect username or password", headers={"WWW-Authenticate": "Basic"})
 
-    def get_script(self, script_name, script_runner):
-        if script_name is None:
+    def get_selectable_script(self, script_name, script_runner):
+        if script_name is None or script_name == "":
             return None, None
 
-        if not script_runner.scripts:
-            script_runner.initialize_scripts(False)
-            ui.create_ui()
-
         script_idx = script_name_to_index(script_name, script_runner.selectable_scripts)
         script = script_runner.selectable_scripts[script_idx]
         return script, script_idx
 
-    def text2imgapi(self, txt2imgreq: StableDiffusionTxt2ImgProcessingAPI):
-        script, script_idx = self.get_script(txt2imgreq.script_name, scripts.scripts_txt2img)
+    def get_scripts_list(self):
+        t2ilist = [script.name for script in scripts.scripts_txt2img.scripts if script.name is not None]
+        i2ilist = [script.name for script in scripts.scripts_img2img.scripts if script.name is not None]
 
-        populate = txt2imgreq.copy(update={ # Override __init__ params
+        return models.ScriptsList(txt2img=t2ilist, img2img=i2ilist)
+
+    def get_script_info(self):
+        res = []
+
+        for script_list in [scripts.scripts_txt2img.scripts, scripts.scripts_img2img.scripts]:
+            res += [script.api_info for script in script_list if script.api_info is not None]
+
+        return res
+
+    def get_script(self, script_name, script_runner):
+        if script_name is None or script_name == "":
+            return None, None
+
+        script_idx = script_name_to_index(script_name, script_runner.scripts)
+        return script_runner.scripts[script_idx]
+
+    def init_default_script_args(self, script_runner):
+        #find max idx from the scripts in runner and generate a none array to init script_args
+        last_arg_index = 1
+        for script in script_runner.scripts:
+            if last_arg_index < script.args_to:
+                last_arg_index = script.args_to
+        # None everywhere except position 0 to initialize script args
+        script_args = [None]*last_arg_index
+        script_args[0] = 0
+
+        # get default values
+        with gr.Blocks(): # will throw errors calling ui function without this
+            for script in script_runner.scripts:
+                if script.ui(script.is_img2img):
+                    ui_default_values = []
+                    for elem in script.ui(script.is_img2img):
+                        ui_default_values.append(elem.value)
+                    script_args[script.args_from:script.args_to] = ui_default_values
+        return script_args
+
+    def init_script_args(self, request, default_script_args, selectable_scripts, selectable_idx, script_runner):
+        script_args = default_script_args.copy()
+        # position 0 in script_arg is the idx+1 of the selectable script that is going to be run when using scripts.scripts_*2img.run()
+        if selectable_scripts:
+            script_args[selectable_scripts.args_from:selectable_scripts.args_to] = request.script_args
+            script_args[0] = selectable_idx + 1
+
+        # Now check for always on scripts
+        if request.alwayson_scripts:
+            for alwayson_script_name in request.alwayson_scripts.keys():
+                alwayson_script = self.get_script(alwayson_script_name, script_runner)
+                if alwayson_script is None:
+                    raise HTTPException(status_code=422, detail=f"always on script {alwayson_script_name} not found")
+                # Selectable script in always on script param check
+                if alwayson_script.alwayson is False:
+                    raise HTTPException(status_code=422, detail="Cannot have a selectable script in the always on scripts params")
+                # always on script with no arg should always run so you don't really need to add them to the requests
+                if "args" in request.alwayson_scripts[alwayson_script_name]:
+                    # min between arg length in scriptrunner and arg length in the request
+                    for idx in range(0, min((alwayson_script.args_to - alwayson_script.args_from), len(request.alwayson_scripts[alwayson_script_name]["args"]))):
+                        script_args[alwayson_script.args_from + idx] = request.alwayson_scripts[alwayson_script_name]["args"][idx]
+        return script_args
+
+    def text2imgapi(self, txt2imgreq: models.StableDiffusionTxt2ImgProcessingAPI):
+        script_runner = scripts.scripts_txt2img
+        if not script_runner.scripts:
+            script_runner.initialize_scripts(False)
+            ui.create_ui()
+        if not self.default_script_arg_txt2img:
+            self.default_script_arg_txt2img = self.init_default_script_args(script_runner)
+        selectable_scripts, selectable_script_idx = self.get_selectable_script(txt2imgreq.script_name, script_runner)
+
+        populate = txt2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(txt2imgreq.sampler_name or txt2imgreq.sampler_index),
-            "do_not_save_samples": True,
-            "do_not_save_grid": True
-            }
-        )
+            "do_not_save_samples": not txt2imgreq.save_images,
+            "do_not_save_grid": not txt2imgreq.save_images,
+        })
         if populate.sampler_name:
             populate.sampler_index = None  # prevent a warning later on
 
         args = vars(populate)
         args.pop('script_name', None)
+        args.pop('script_args', None) # will refeed them to the pipeline directly after initializing them
+        args.pop('alwayson_scripts', None)
+
+        script_args = self.init_script_args(txt2imgreq, self.default_script_arg_txt2img, selectable_scripts, selectable_script_idx, script_runner)
+
+        send_images = args.pop('send_images', True)
+        args.pop('save_images', None)
 
         with self.queue_lock:
             p = StableDiffusionProcessingTxt2Img(sd_model=shared.sd_model, **args)
+            p.scripts = script_runner
+            p.outpath_grids = opts.outdir_txt2img_grids
+            p.outpath_samples = opts.outdir_txt2img_samples
 
             shared.state.begin()
-            if script is not None:
-                p.outpath_grids = opts.outdir_txt2img_grids
-                p.outpath_samples = opts.outdir_txt2img_samples
-                p.script_args = [script_idx + 1] + [None] * (script.args_from - 1) + p.script_args
-                processed = scripts.scripts_txt2img.run(p, *p.script_args)
+            if selectable_scripts is not None:
+                p.script_args = script_args
+                processed = scripts.scripts_txt2img.run(p, *p.script_args) # Need to pass args as list here
             else:
+                p.script_args = tuple(script_args) # Need to pass args as tuple here
                 processed = process_images(p)
             shared.state.end()
 
-        b64images = list(map(encode_pil_to_base64, processed.images))
+        b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else []
 
-        return TextToImageResponse(images=b64images, parameters=vars(txt2imgreq), info=processed.js())
+        return models.TextToImageResponse(images=b64images, parameters=vars(txt2imgreq), info=processed.js())
 
-    def img2imgapi(self, img2imgreq: StableDiffusionImg2ImgProcessingAPI):
+    def img2imgapi(self, img2imgreq: models.StableDiffusionImg2ImgProcessingAPI):
         init_images = img2imgreq.init_images
         if init_images is None:
             raise HTTPException(status_code=404, detail="Init image not found")
 
-        script, script_idx = self.get_script(img2imgreq.script_name, scripts.scripts_img2img)
-
         mask = img2imgreq.mask
         if mask:
             mask = decode_base64_to_image(mask)
 
-        populate = img2imgreq.copy(update={ # Override __init__ params
+        script_runner = scripts.scripts_img2img
+        if not script_runner.scripts:
+            script_runner.initialize_scripts(True)
+            ui.create_ui()
+        if not self.default_script_arg_img2img:
+            self.default_script_arg_img2img = self.init_default_script_args(script_runner)
+        selectable_scripts, selectable_script_idx = self.get_selectable_script(img2imgreq.script_name, script_runner)
+
+        populate = img2imgreq.copy(update={  # Override __init__ params
             "sampler_name": validate_sampler_name(img2imgreq.sampler_name or img2imgreq.sampler_index),
-            "do_not_save_samples": True,
-            "do_not_save_grid": True,
-            "mask": mask
-            }
-        )
+            "do_not_save_samples": not img2imgreq.save_images,
+            "do_not_save_grid": not img2imgreq.save_images,
+            "mask": mask,
+        })
         if populate.sampler_name:
             populate.sampler_index = None  # prevent a warning later on
 
         args = vars(populate)
         args.pop('include_init_images', None)  # this is meant to be done by "exclude": True in model, but it's for a reason that I cannot determine.
         args.pop('script_name', None)
+        args.pop('script_args', None)  # will refeed them to the pipeline directly after initializing them
+        args.pop('alwayson_scripts', None)
+
+        script_args = self.init_script_args(img2imgreq, self.default_script_arg_img2img, selectable_scripts, selectable_script_idx, script_runner)
+
+        send_images = args.pop('send_images', True)
+        args.pop('save_images', None)
 
         with self.queue_lock:
             p = StableDiffusionProcessingImg2Img(sd_model=shared.sd_model, **args)
             p.init_images = [decode_base64_to_image(x) for x in init_images]
+            p.scripts = script_runner
+            p.outpath_grids = opts.outdir_img2img_grids
+            p.outpath_samples = opts.outdir_img2img_samples
 
             shared.state.begin()
-            if script is not None:
-                p.outpath_grids = opts.outdir_img2img_grids
-                p.outpath_samples = opts.outdir_img2img_samples
-                p.script_args = [script_idx + 1] + [None] * (script.args_from - 1) + p.script_args
-                processed = scripts.scripts_img2img.run(p, *p.script_args)
+            if selectable_scripts is not None:
+                p.script_args = script_args
+                processed = scripts.scripts_img2img.run(p, *p.script_args) # Need to pass args as list here
             else:
+                p.script_args = tuple(script_args) # Need to pass args as tuple here
                 processed = process_images(p)
             shared.state.end()
 
-        b64images = list(map(encode_pil_to_base64, processed.images))
+        b64images = list(map(encode_pil_to_base64, processed.images)) if send_images else []
 
         if not img2imgreq.include_init_images:
             img2imgreq.init_images = None
             img2imgreq.mask = None
 
-        return ImageToImageResponse(images=b64images, parameters=vars(img2imgreq), info=processed.js())
+        return models.ImageToImageResponse(images=b64images, parameters=vars(img2imgreq), info=processed.js())
 
-    def extras_single_image_api(self, req: ExtrasSingleImageRequest):
+    def extras_single_image_api(self, req: models.ExtrasSingleImageRequest):
         reqDict = setUpscalers(req)
 
         reqDict['image'] = decode_base64_to_image(reqDict['image'])
@@ -261,31 +411,26 @@ class Api:
         with self.queue_lock:
             result = postprocessing.run_extras(extras_mode=0, image_folder="", input_dir="", output_dir="", save_output=False, **reqDict)
 
-        return ExtrasSingleImageResponse(image=encode_pil_to_base64(result[0][0]), html_info=result[1])
+        return models.ExtrasSingleImageResponse(image=encode_pil_to_base64(result[0][0]), html_info=result[1])
 
-    def extras_batch_images_api(self, req: ExtrasBatchImagesRequest):
+    def extras_batch_images_api(self, req: models.ExtrasBatchImagesRequest):
         reqDict = setUpscalers(req)
 
-        def prepareFiles(file):
-            file = decode_base64_to_file(file.data, file_path=file.name)
-            file.orig_name = file.name
-            return file
-
-        reqDict['image_folder'] = list(map(prepareFiles, reqDict['imageList']))
-        reqDict.pop('imageList')
+        image_list = reqDict.pop('imageList', [])
+        image_folder = [decode_base64_to_image(x.data) for x in image_list]
 
         with self.queue_lock:
-            result = postprocessing.run_extras(extras_mode=1, image="", input_dir="", output_dir="", save_output=False, **reqDict)
+            result = postprocessing.run_extras(extras_mode=1, image_folder=image_folder, image="", input_dir="", output_dir="", save_output=False, **reqDict)
 
-        return ExtrasBatchImagesResponse(images=list(map(encode_pil_to_base64, result[0])), html_info=result[1])
+        return models.ExtrasBatchImagesResponse(images=list(map(encode_pil_to_base64, result[0])), html_info=result[1])
 
-    def pnginfoapi(self, req: PNGInfoRequest):
+    def pnginfoapi(self, req: models.PNGInfoRequest):
         if(not req.image.strip()):
-            return PNGInfoResponse(info="")
+            return models.PNGInfoResponse(info="")
 
         image = decode_base64_to_image(req.image.strip())
         if image is None:
-            return PNGInfoResponse(info="")
+            return models.PNGInfoResponse(info="")
 
         geninfo, items = images.read_info_from_image(image)
         if geninfo is None:
@@ -293,13 +438,13 @@ class Api:
 
         items = {**{'parameters': geninfo}, **items}
 
-        return PNGInfoResponse(info=geninfo, items=items)
+        return models.PNGInfoResponse(info=geninfo, items=items)
 
-    def progressapi(self, req: ProgressRequest = Depends()):
+    def progressapi(self, req: models.ProgressRequest = Depends()):
         # copy from check_progress_call of ui.py
 
         if shared.state.job_count == 0:
-            return ProgressResponse(progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo)
+            return models.ProgressResponse(progress=0, eta_relative=0, state=shared.state.dict(), textinfo=shared.state.textinfo)
 
         # avoid dividing zero
         progress = 0.01
@@ -321,9 +466,9 @@ class Api:
         if shared.state.current_image and not req.skip_current_image:
             current_image = encode_pil_to_base64(shared.state.current_image)
 
-        return ProgressResponse(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo)
+        return models.ProgressResponse(progress=progress, eta_relative=eta_relative, state=shared.state.dict(), current_image=current_image, textinfo=shared.state.textinfo)
 
-    def interrogateapi(self, interrogatereq: InterrogateRequest):
+    def interrogateapi(self, interrogatereq: models.InterrogateRequest):
         image_b64 = interrogatereq.image
         if image_b64 is None:
             raise HTTPException(status_code=404, detail="Image not found")
@@ -340,13 +485,23 @@ class Api:
             else:
                 raise HTTPException(status_code=404, detail="Model not found")
 
-        return InterrogateResponse(caption=processed)
+        return models.InterrogateResponse(caption=processed)
 
     def interruptapi(self):
         shared.state.interrupt()
 
         return {}
 
+    def unloadapi(self):
+        unload_model_weights()
+
+        return {}
+
+    def reloadapi(self):
+        reload_model_weights()
+
+        return {}
+
     def skip(self):
         shared.state.skip()
 
@@ -386,8 +541,19 @@ class Api:
             for upscaler in shared.sd_upscalers
         ]
 
+    def get_latent_upscale_modes(self):
+        return [
+            {
+                "name": upscale_mode,
+            }
+            for upscale_mode in [*(shared.latent_upscale_modes or {})]
+        ]
+
     def get_sd_models(self):
-        return [{"title": x.title, "model_name": x.model_name, "hash": x.shorthash, "sha256": x.sha256, "filename": x.filename, "config": find_checkpoint_config(x)} for x in checkpoints_list.values()]
+        return [{"title": x.title, "model_name": x.model_name, "hash": x.shorthash, "sha256": x.sha256, "filename": x.filename, "config": find_checkpoint_config_near_filename(x)} for x in checkpoints_list.values()]
+
+    def get_sd_vaes(self):
+        return [{"model_name": x, "filename": vae_dict[x]} for x in vae_dict.keys()]
 
     def get_hypernetworks(self):
         return [{"name": name, "path": shared.hypernetworks[name]} for name in shared.hypernetworks]
@@ -435,36 +601,36 @@ class Api:
             filename = create_embedding(**args) # create empty embedding
             sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings() # reload embeddings so new one can be immediately used
             shared.state.end()
-            return CreateResponse(info = "create embedding filename: {filename}".format(filename = filename))
+            return models.CreateResponse(info=f"create embedding filename: {filename}")
         except AssertionError as e:
             shared.state.end()
-            return TrainResponse(info = "create embedding error: {error}".format(error = e))
+            return models.TrainResponse(info=f"create embedding error: {e}")
 
     def create_hypernetwork(self, args: dict):
         try:
             shared.state.begin()
             filename = create_hypernetwork(**args) # create empty embedding
             shared.state.end()
-            return CreateResponse(info = "create hypernetwork filename: {filename}".format(filename = filename))
+            return models.CreateResponse(info=f"create hypernetwork filename: {filename}")
         except AssertionError as e:
             shared.state.end()
-            return TrainResponse(info = "create hypernetwork error: {error}".format(error = e))
+            return models.TrainResponse(info=f"create hypernetwork error: {e}")
 
     def preprocess(self, args: dict):
         try:
             shared.state.begin()
             preprocess(**args) # quick operation unless blip/booru interrogation is enabled
             shared.state.end()
-            return PreprocessResponse(info = 'preprocess complete')
+            return models.PreprocessResponse(info = 'preprocess complete')
         except KeyError as e:
             shared.state.end()
-            return PreprocessResponse(info = "preprocess error: invalid token: {error}".format(error = e))
+            return models.PreprocessResponse(info=f"preprocess error: invalid token: {e}")
         except AssertionError as e:
             shared.state.end()
-            return PreprocessResponse(info = "preprocess error: {error}".format(error = e))
+            return models.PreprocessResponse(info=f"preprocess error: {e}")
         except FileNotFoundError as e:
             shared.state.end()
-            return PreprocessResponse(info = 'preprocess error: {error}'.format(error = e))
+            return models.PreprocessResponse(info=f'preprocess error: {e}')
 
     def train_embedding(self, args: dict):
         try:
@@ -482,10 +648,10 @@ class Api:
                 if not apply_optimizations:
                     sd_hijack.apply_optimizations()
                 shared.state.end()
-            return TrainResponse(info = "train embedding complete: filename: {filename} error: {error}".format(filename = filename, error = error))
+            return models.TrainResponse(info=f"train embedding complete: filename: {filename} error: {error}")
         except AssertionError as msg:
             shared.state.end()
-            return TrainResponse(info = "train embedding error: {msg}".format(msg = msg))
+            return models.TrainResponse(info=f"train embedding error: {msg}")
 
     def train_hypernetwork(self, args: dict):
         try:
@@ -497,7 +663,7 @@ class Api:
             if not apply_optimizations:
                 sd_hijack.undo_optimizations()
             try:
-                hypernetwork, filename = train_hypernetwork(*args)
+                hypernetwork, filename = train_hypernetwork(**args)
             except Exception as e:
                 error = e
             finally:
@@ -506,14 +672,15 @@ class Api:
                 if not apply_optimizations:
                     sd_hijack.apply_optimizations()
                 shared.state.end()
-            return TrainResponse(info="train embedding complete: filename: {filename} error: {error}".format(filename=filename, error=error))
-        except AssertionError as msg:
+            return models.TrainResponse(info=f"train embedding complete: filename: {filename} error: {error}")
+        except AssertionError:
             shared.state.end()
-            return TrainResponse(info="train embedding error: {error}".format(error=error))
+            return models.TrainResponse(info=f"train embedding error: {error}")
 
     def get_memory(self):
         try:
-            import os, psutil
+            import os
+            import psutil
             process = psutil.Process(os.getpid())
             res = process.memory_info() # only rss is cross-platform guaranteed so we dont rely on other values
             ram_total = 100 * res.rss / process.memory_percent() # and total memory is calculated as actual value is not cross-platform safe
@@ -540,11 +707,11 @@ class Api:
                     'events': warnings,
                 }
             else:
-                cuda = { 'error': 'unavailable' }
+                cuda = {'error': 'unavailable'}
         except Exception as err:
-            cuda = { 'error': f'{err}' }
-        return MemoryResponse(ram = ram, cuda = cuda)
+            cuda = {'error': f'{err}'}
+        return models.MemoryResponse(ram=ram, cuda=cuda)
 
     def launch(self, server_name, port):
         self.app.include_router(self.router)
-        uvicorn.run(self.app, host=server_name, port=port)
+        uvicorn.run(self.app, host=server_name, port=port, timeout_keep_alive=0)

modules/api/models.py

@@ -14,8 +14,8 @@ API_NOT_ALLOWED = [
     "outpath_samples",
     "outpath_grids",
     "sampler_index",
-    "do_not_save_samples",
-    "do_not_save_grid",
+    # "do_not_save_samples",
+    # "do_not_save_grid",
     "extra_generation_params",
     "overlay_images",
     "do_not_reload_embeddings",
@@ -100,13 +100,31 @@ class PydanticModelGenerator:
 StableDiffusionTxt2ImgProcessingAPI = PydanticModelGenerator(
     "StableDiffusionProcessingTxt2Img",
     StableDiffusionProcessingTxt2Img,
-    [{"key": "sampler_index", "type": str, "default": "Euler"}, {"key": "script_name", "type": str, "default": None}, {"key": "script_args", "type": list, "default": []}]
+    [
+        {"key": "sampler_index", "type": str, "default": "Euler"},
+        {"key": "script_name", "type": str, "default": None},
+        {"key": "script_args", "type": list, "default": []},
+        {"key": "send_images", "type": bool, "default": True},
+        {"key": "save_images", "type": bool, "default": False},
+        {"key": "alwayson_scripts", "type": dict, "default": {}},
+    ]
 ).generate_model()
 
 StableDiffusionImg2ImgProcessingAPI = PydanticModelGenerator(
     "StableDiffusionProcessingImg2Img",
     StableDiffusionProcessingImg2Img,
-    [{"key": "sampler_index", "type": str, "default": "Euler"}, {"key": "init_images", "type": list, "default": None}, {"key": "denoising_strength", "type": float, "default": 0.75}, {"key": "mask", "type": str, "default": None}, {"key": "include_init_images", "type": bool, "default": False, "exclude" : True}, {"key": "script_name", "type": str, "default": None}, {"key": "script_args", "type": list, "default": []}]
+    [
+        {"key": "sampler_index", "type": str, "default": "Euler"},
+        {"key": "init_images", "type": list, "default": None},
+        {"key": "denoising_strength", "type": float, "default": 0.75},
+        {"key": "mask", "type": str, "default": None},
+        {"key": "include_init_images", "type": bool, "default": False, "exclude" : True},
+        {"key": "script_name", "type": str, "default": None},
+        {"key": "script_args", "type": list, "default": []},
+        {"key": "send_images", "type": bool, "default": True},
+        {"key": "save_images", "type": bool, "default": False},
+        {"key": "alwayson_scripts", "type": dict, "default": {}},
+    ]
 ).generate_model()
 
 class TextToImageResponse(BaseModel):
@@ -205,8 +223,9 @@ for key in _options:
     if(_options[key].dest != 'help'):
         flag = _options[key]
         _type = str
-        if _options[key].default is not None: _type = type(_options[key].default)
-        flags.update({flag.dest: (_type,Field(default=flag.default, description=flag.help))})
+        if _options[key].default is not None:
+            _type = type(_options[key].default)
+        flags.update({flag.dest: (_type, Field(default=flag.default, description=flag.help))})
 
 FlagsModel = create_model("Flags", **flags)
 
@@ -222,13 +241,20 @@ class UpscalerItem(BaseModel):
     model_url: Optional[str] = Field(title="URL")
     scale: Optional[float] = Field(title="Scale")
 
+class LatentUpscalerModeItem(BaseModel):
+    name: str = Field(title="Name")
+
 class SDModelItem(BaseModel):
     title: str = Field(title="Title")
     model_name: str = Field(title="Model Name")
     hash: Optional[str] = Field(title="Short hash")
     sha256: Optional[str] = Field(title="sha256 hash")
     filename: str = Field(title="Filename")
-    config: str = Field(title="Config file")
+    config: Optional[str] = Field(title="Config file")
+
+class SDVaeItem(BaseModel):
+    model_name: str = Field(title="Model Name")
+    filename: str = Field(title="Filename")
 
 class HypernetworkItem(BaseModel):
     name: str = Field(title="Name")
@@ -267,3 +293,24 @@ class EmbeddingsResponse(BaseModel):
 class MemoryResponse(BaseModel):
     ram: dict = Field(title="RAM", description="System memory stats")
     cuda: dict = Field(title="CUDA", description="nVidia CUDA memory stats")
+
+
+class ScriptsList(BaseModel):
+    txt2img: list = Field(default=None, title="Txt2img", description="Titles of scripts (txt2img)")
+    img2img: list = Field(default=None, title="Img2img", description="Titles of scripts (img2img)")
+
+
+class ScriptArg(BaseModel):
+    label: str = Field(default=None, title="Label", description="Name of the argument in UI")
+    value: Optional[Any] = Field(default=None, title="Value", description="Default value of the argument")
+    minimum: Optional[Any] = Field(default=None, title="Minimum", description="Minimum allowed value for the argumentin UI")
+    maximum: Optional[Any] = Field(default=None, title="Minimum", description="Maximum allowed value for the argumentin UI")
+    step: Optional[Any] = Field(default=None, title="Minimum", description="Step for changing value of the argumentin UI")
+    choices: Optional[List[str]] = Field(default=None, title="Choices", description="Possible values for the argument")
+
+
+class ScriptInfo(BaseModel):
+    name: str = Field(default=None, title="Name", description="Script name")
+    is_alwayson: bool = Field(default=None, title="IsAlwayson", description="Flag specifying whether this script is an alwayson script")
+    is_img2img: bool = Field(default=None, title="IsImg2img", description="Flag specifying whether this script is an img2img script")
+    args: List[ScriptArg] = Field(title="Arguments", description="List of script's arguments")

modules/call_queue.py

@@ -1,10 +1,8 @@
 import html
-import sys
 import threading
-import traceback
 import time
 
-from modules import shared, progress
+from modules import shared, progress, errors
 
 queue_lock = threading.Lock()
 
@@ -23,7 +21,7 @@ def wrap_gradio_gpu_call(func, extra_outputs=None):
     def f(*args, **kwargs):
 
         # if the first argument is a string that says "task(...)", it is treated as a job id
-        if len(args) > 0 and type(args[0]) == str and args[0][0:5] == "task(" and args[0][-1] == ")":
+        if args and type(args[0]) == str and args[0].startswith("task(") and args[0].endswith(")"):
             id_task = args[0]
             progress.add_task_to_queue(id_task)
         else:
@@ -35,6 +33,7 @@ def wrap_gradio_gpu_call(func, extra_outputs=None):
 
             try:
                 res = func(*args, **kwargs)
+                progress.record_results(id_task, res)
             finally:
                 progress.finish_task(id_task)
 
@@ -55,16 +54,14 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False):
         try:
             res = list(func(*args, **kwargs))
         except Exception as e:
-            # When printing out our debug argument list, do not print out more than a MB of text
-            max_debug_str_len = 131072 # (1024*1024)/8
-
-            print("Error completing request", file=sys.stderr)
-            argStr = f"Arguments: {str(args)} {str(kwargs)}"
-            print(argStr[:max_debug_str_len], file=sys.stderr)
-            if len(argStr) > max_debug_str_len:
-                print(f"(Argument list truncated at {max_debug_str_len}/{len(argStr)} characters)", file=sys.stderr)
-
-            print(traceback.format_exc(), file=sys.stderr)
+            # When printing out our debug argument list,
+            # do not print out more than a 100 KB of text
+            max_debug_str_len = 131072
+            message = "Error completing request"
+            arg_str = f"Arguments: {args} {kwargs}"[:max_debug_str_len]
+            if len(arg_str) > max_debug_str_len:
+                arg_str += f" (Argument list truncated at {max_debug_str_len}/{len(arg_str)} characters)"
+            errors.report(f"{message}\n{arg_str}", exc_info=True)
 
             shared.state.job = ""
             shared.state.job_count = 0
@@ -72,7 +69,8 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False):
             if extra_outputs_array is None:
                 extra_outputs_array = [None, '']
 
-            res = extra_outputs_array + [f"<div class='error'>{html.escape(type(e).__name__+': '+str(e))}</div>"]
+            error_message = f'{type(e).__name__}: {e}'
+            res = extra_outputs_array + [f"<div class='error'>{html.escape(error_message)}</div>"]
 
         shared.state.skipped = False
         shared.state.interrupted = False
@@ -106,4 +104,3 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False):
         return tuple(res)
 
     return f
-

modules/cmd_args.py

@@ -0,0 +1,109 @@
+import argparse
+import json
+import os
+from modules.paths_internal import models_path, script_path, data_path, extensions_dir, extensions_builtin_dir, sd_default_config, sd_model_file  # noqa: F401
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-f", action='store_true', help=argparse.SUPPRESS)  # allows running as root; implemented outside of webui
+parser.add_argument("--update-all-extensions", action='store_true', help="launch.py argument: download updates for all extensions when starting the program")
+parser.add_argument("--skip-python-version-check", action='store_true', help="launch.py argument: do not check python version")
+parser.add_argument("--skip-torch-cuda-test", action='store_true', help="launch.py argument: do not check if CUDA is able to work properly")
+parser.add_argument("--reinstall-xformers", action='store_true', help="launch.py argument: install the appropriate version of xformers even if you have some version already installed")
+parser.add_argument("--reinstall-torch", action='store_true', help="launch.py argument: install the appropriate version of torch even if you have some version already installed")
+parser.add_argument("--update-check", action='store_true', help="launch.py argument: check for updates at startup")
+parser.add_argument("--test-server", action='store_true', help="launch.py argument: configure server for testing")
+parser.add_argument("--skip-prepare-environment", action='store_true', help="launch.py argument: skip all environment preparation")
+parser.add_argument("--skip-install", action='store_true', help="launch.py argument: skip installation of packages")
+parser.add_argument("--data-dir", type=str, default=os.path.dirname(os.path.dirname(os.path.realpath(__file__))), help="base path where all user data is stored")
+parser.add_argument("--config", type=str, default=sd_default_config, help="path to config which constructs model",)
+parser.add_argument("--ckpt", type=str, default=sd_model_file, help="path to checkpoint of stable diffusion model; if specified, this checkpoint will be added to the list of checkpoints and loaded",)
+parser.add_argument("--ckpt-dir", type=str, default=None, help="Path to directory with stable diffusion checkpoints")
+parser.add_argument("--vae-dir", type=str, default=None, help="Path to directory with VAE files")
+parser.add_argument("--gfpgan-dir", type=str, help="GFPGAN directory", default=('./src/gfpgan' if os.path.exists('./src/gfpgan') else './GFPGAN'))
+parser.add_argument("--gfpgan-model", type=str, help="GFPGAN model file name", default=None)
+parser.add_argument("--no-half", action='store_true', help="do not switch the model to 16-bit floats")
+parser.add_argument("--no-half-vae", action='store_true', help="do not switch the VAE model to 16-bit floats")
+parser.add_argument("--no-progressbar-hiding", action='store_true', help="do not hide progressbar in gradio UI (we hide it because it slows down ML if you have hardware acceleration in browser)")
+parser.add_argument("--max-batch-count", type=int, default=16, help="maximum batch count value for the UI")
+parser.add_argument("--embeddings-dir", type=str, default=os.path.join(data_path, 'embeddings'), help="embeddings directory for textual inversion (default: embeddings)")
+parser.add_argument("--textual-inversion-templates-dir", type=str, default=os.path.join(script_path, 'textual_inversion_templates'), help="directory with textual inversion templates")
+parser.add_argument("--hypernetwork-dir", type=str, default=os.path.join(models_path, 'hypernetworks'), help="hypernetwork directory")
+parser.add_argument("--localizations-dir", type=str, default=os.path.join(script_path, 'localizations'), help="localizations directory")
+parser.add_argument("--allow-code", action='store_true', help="allow custom script execution from webui")
+parser.add_argument("--medvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a little speed for low VRM usage")
+parser.add_argument("--lowvram", action='store_true', help="enable stable diffusion model optimizations for sacrificing a lot of speed for very low VRM usage")
+parser.add_argument("--lowram", action='store_true', help="load stable diffusion checkpoint weights to VRAM instead of RAM")
+parser.add_argument("--always-batch-cond-uncond", action='store_true', help="disables cond/uncond batching that is enabled to save memory with --medvram or --lowvram")
+parser.add_argument("--unload-gfpgan", action='store_true', help="does not do anything.")
+parser.add_argument("--precision", type=str, help="evaluate at this precision", choices=["full", "autocast"], default="autocast")
+parser.add_argument("--upcast-sampling", action='store_true', help="upcast sampling. No effect with --no-half. Usually produces similar results to --no-half with better performance while using less memory.")
+parser.add_argument("--share", action='store_true', help="use share=True for gradio and make the UI accessible through their site")
+parser.add_argument("--ngrok", type=str, help="ngrok authtoken, alternative to gradio --share", default=None)
+parser.add_argument("--ngrok-region", type=str, help="does not do anything.", default="")
+parser.add_argument("--ngrok-options", type=json.loads, help='The options to pass to ngrok in JSON format, e.g.: \'{"authtoken_from_env":true, "basic_auth":"user:password", "oauth_provider":"google", "oauth_allow_emails":"user@asdf.com"}\'', default=dict())
+parser.add_argument("--enable-insecure-extension-access", action='store_true', help="enable extensions tab regardless of other options")
+parser.add_argument("--codeformer-models-path", type=str, help="Path to directory with codeformer model file(s).", default=os.path.join(models_path, 'Codeformer'))
+parser.add_argument("--gfpgan-models-path", type=str, help="Path to directory with GFPGAN model file(s).", default=os.path.join(models_path, 'GFPGAN'))
+parser.add_argument("--esrgan-models-path", type=str, help="Path to directory with ESRGAN model file(s).", default=os.path.join(models_path, 'ESRGAN'))
+parser.add_argument("--bsrgan-models-path", type=str, help="Path to directory with BSRGAN model file(s).", default=os.path.join(models_path, 'BSRGAN'))
+parser.add_argument("--realesrgan-models-path", type=str, help="Path to directory with RealESRGAN model file(s).", default=os.path.join(models_path, 'RealESRGAN'))
+parser.add_argument("--clip-models-path", type=str, help="Path to directory with CLIP model file(s).", default=None)
+parser.add_argument("--xformers", action='store_true', help="enable xformers for cross attention layers")
+parser.add_argument("--force-enable-xformers", action='store_true', help="enable xformers for cross attention layers regardless of whether the checking code thinks you can run it; do not make bug reports if this fails to work")
+parser.add_argument("--xformers-flash-attention", action='store_true', help="enable xformers with Flash Attention to improve reproducibility (supported for SD2.x or variant only)")
+parser.add_argument("--deepdanbooru", action='store_true', help="does not do anything")
+parser.add_argument("--opt-split-attention", action='store_true', help="prefer Doggettx's cross-attention layer optimization for automatic choice of optimization")
+parser.add_argument("--opt-sub-quad-attention", action='store_true', help="prefer memory efficient sub-quadratic cross-attention layer optimization for automatic choice of optimization")
+parser.add_argument("--sub-quad-q-chunk-size", type=int, help="query chunk size for the sub-quadratic cross-attention layer optimization to use", default=1024)
+parser.add_argument("--sub-quad-kv-chunk-size", type=int, help="kv chunk size for the sub-quadratic cross-attention layer optimization to use", default=None)
+parser.add_argument("--sub-quad-chunk-threshold", type=int, help="the percentage of VRAM threshold for the sub-quadratic cross-attention layer optimization to use chunking", default=None)
+parser.add_argument("--opt-split-attention-invokeai", action='store_true', help="prefer InvokeAI's cross-attention layer optimization for automatic choice of optimization")
+parser.add_argument("--opt-split-attention-v1", action='store_true', help="prefer older version of split attention optimization for automatic choice of optimization")
+parser.add_argument("--opt-sdp-attention", action='store_true', help="prefer scaled dot product cross-attention layer optimization for automatic choice of optimization; requires PyTorch 2.*")
+parser.add_argument("--opt-sdp-no-mem-attention", action='store_true', help="prefer scaled dot product cross-attention layer optimization without memory efficient attention for automatic choice of optimization, makes image generation deterministic; requires PyTorch 2.*")
+parser.add_argument("--disable-opt-split-attention", action='store_true', help="prefer no cross-attention layer optimization for automatic choice of optimization")
+parser.add_argument("--disable-nan-check", action='store_true', help="do not check if produced images/latent spaces have nans; useful for running without a checkpoint in CI")
+parser.add_argument("--use-cpu", nargs='+', help="use CPU as torch device for specified modules", default=[], type=str.lower)
+parser.add_argument("--listen", action='store_true', help="launch gradio with 0.0.0.0 as server name, allowing to respond to network requests")
+parser.add_argument("--port", type=int, help="launch gradio with given server port, you need root/admin rights for ports < 1024, defaults to 7860 if available", default=None)
+parser.add_argument("--show-negative-prompt", action='store_true', help="does not do anything", default=False)
+parser.add_argument("--ui-config-file", type=str, help="filename to use for ui configuration", default=os.path.join(data_path, 'ui-config.json'))
+parser.add_argument("--hide-ui-dir-config", action='store_true', help="hide directory configuration from webui", default=False)
+parser.add_argument("--freeze-settings", action='store_true', help="disable editing settings", default=False)
+parser.add_argument("--ui-settings-file", type=str, help="filename to use for ui settings", default=os.path.join(data_path, 'config.json'))
+parser.add_argument("--gradio-debug",  action='store_true', help="launch gradio with --debug option")
+parser.add_argument("--gradio-auth", type=str, help='set gradio authentication like "username:password"; or comma-delimit multiple like "u1:p1,u2:p2,u3:p3"', default=None)
+parser.add_argument("--gradio-auth-path", type=str, help='set gradio authentication file path ex. "/path/to/auth/file" same auth format as --gradio-auth', default=None)
+parser.add_argument("--gradio-img2img-tool", type=str, help='does not do anything')
+parser.add_argument("--gradio-inpaint-tool", type=str, help="does not do anything")
+parser.add_argument("--gradio-allowed-path", action='append', help="add path to gradio's allowed_paths, make it possible to serve files from it")
+parser.add_argument("--opt-channelslast", action='store_true', help="change memory type for stable diffusion to channels last")
+parser.add_argument("--styles-file", type=str, help="filename to use for styles", default=os.path.join(data_path, 'styles.csv'))
+parser.add_argument("--autolaunch", action='store_true', help="open the webui URL in the system's default browser upon launch", default=False)
+parser.add_argument("--theme", type=str, help="launches the UI with light or dark theme", default=None)
+parser.add_argument("--use-textbox-seed", action='store_true', help="use textbox for seeds in UI (no up/down, but possible to input long seeds)", default=False)
+parser.add_argument("--disable-console-progressbars", action='store_true', help="do not output progressbars to console", default=False)
+parser.add_argument("--enable-console-prompts", action='store_true', help="print prompts to console when generating with txt2img and img2img", default=False)
+parser.add_argument('--vae-path', type=str, help='Checkpoint to use as VAE; setting this argument disables all settings related to VAE', default=None)
+parser.add_argument("--disable-safe-unpickle", action='store_true', help="disable checking pytorch models for malicious code", default=False)
+parser.add_argument("--api", action='store_true', help="use api=True to launch the API together with the webui (use --nowebui instead for only the API)")
+parser.add_argument("--api-auth", type=str, help='Set authentication for API like "username:password"; or comma-delimit multiple like "u1:p1,u2:p2,u3:p3"', default=None)
+parser.add_argument("--api-log", action='store_true', help="use api-log=True to enable logging of all API requests")
+parser.add_argument("--nowebui", action='store_true', help="use api=True to launch the API instead of the webui")
+parser.add_argument("--ui-debug-mode", action='store_true', help="Don't load model to quickly launch UI")
+parser.add_argument("--device-id", type=str, help="Select the default CUDA device to use (export CUDA_VISIBLE_DEVICES=0,1,etc might be needed before)", default=None)
+parser.add_argument("--administrator", action='store_true', help="Administrator rights", default=False)
+parser.add_argument("--cors-allow-origins", type=str, help="Allowed CORS origin(s) in the form of a comma-separated list (no spaces)", default=None)
+parser.add_argument("--cors-allow-origins-regex", type=str, help="Allowed CORS origin(s) in the form of a single regular expression", default=None)
+parser.add_argument("--tls-keyfile", type=str, help="Partially enables TLS, requires --tls-certfile to fully function", default=None)
+parser.add_argument("--tls-certfile", type=str, help="Partially enables TLS, requires --tls-keyfile to fully function", default=None)
+parser.add_argument("--disable-tls-verify", action="store_false", help="When passed, enables the use of self-signed certificates.", default=None)
+parser.add_argument("--server-name", type=str, help="Sets hostname of server", default=None)
+parser.add_argument("--gradio-queue", action='store_true', help="does not do anything", default=True)
+parser.add_argument("--no-gradio-queue", action='store_true', help="Disables gradio queue; causes the webpage to use http requests instead of websockets; was the defaul in earlier versions")
+parser.add_argument("--skip-version-check", action='store_true', help="Do not check versions of torch and xformers")
+parser.add_argument("--no-hashing", action='store_true', help="disable sha256 hashing of checkpoints to help loading performance", default=False)
+parser.add_argument("--no-download-sd-model", action='store_true', help="don't download SD1.5 model even if no model is found in --ckpt-dir", default=False)
+parser.add_argument('--subpath', type=str, help='customize the subpath for gradio, use with reverse proxy')
+parser.add_argument('--add-stop-route', action='store_true', help='add /_stop route to stop server')

modules/codeformer/codeformer_arch.py

@@ -1,14 +1,12 @@
 # this file is copied from CodeFormer repository. Please see comment in modules/codeformer_model.py
 
 import math
-import numpy as np
 import torch
 from torch import nn, Tensor
 import torch.nn.functional as F
-from typing import Optional, List
+from typing import Optional
 
-from modules.codeformer.vqgan_arch import *
-from basicsr.utils import get_root_logger
+from modules.codeformer.vqgan_arch import VQAutoEncoder, ResBlock
 from basicsr.utils.registry import ARCH_REGISTRY
 
 def calc_mean_std(feat, eps=1e-5):
@@ -121,7 +119,7 @@ class TransformerSALayer(nn.Module):
                 tgt_mask: Optional[Tensor] = None,
                 tgt_key_padding_mask: Optional[Tensor] = None,
                 query_pos: Optional[Tensor] = None):
-        
+
         # self attention
         tgt2 = self.norm1(tgt)
         q = k = self.with_pos_embed(tgt2, query_pos)
@@ -161,10 +159,10 @@ class Fuse_sft_block(nn.Module):
 
 @ARCH_REGISTRY.register()
 class CodeFormer(VQAutoEncoder):
-    def __init__(self, dim_embd=512, n_head=8, n_layers=9, 
+    def __init__(self, dim_embd=512, n_head=8, n_layers=9,
                 codebook_size=1024, latent_size=256,
-                connect_list=['32', '64', '128', '256'],
-                fix_modules=['quantize','generator']):
+                connect_list=('32', '64', '128', '256'),
+                fix_modules=('quantize', 'generator')):
         super(CodeFormer, self).__init__(512, 64, [1, 2, 2, 4, 4, 8], 'nearest',2, [16], codebook_size)
 
         if fix_modules is not None:
@@ -181,14 +179,14 @@ class CodeFormer(VQAutoEncoder):
         self.feat_emb = nn.Linear(256, self.dim_embd)
 
         # transformer
-        self.ft_layers = nn.Sequential(*[TransformerSALayer(embed_dim=dim_embd, nhead=n_head, dim_mlp=self.dim_mlp, dropout=0.0) 
+        self.ft_layers = nn.Sequential(*[TransformerSALayer(embed_dim=dim_embd, nhead=n_head, dim_mlp=self.dim_mlp, dropout=0.0)
                                     for _ in range(self.n_layers)])
 
         # logits_predict head
         self.idx_pred_layer = nn.Sequential(
             nn.LayerNorm(dim_embd),
             nn.Linear(dim_embd, codebook_size, bias=False))
-        
+
         self.channels = {
             '16': 512,
             '32': 256,
@@ -223,7 +221,7 @@ class CodeFormer(VQAutoEncoder):
         enc_feat_dict = {}
         out_list = [self.fuse_encoder_block[f_size] for f_size in self.connect_list]
         for i, block in enumerate(self.encoder.blocks):
-            x = block(x) 
+            x = block(x)
             if i in out_list:
                 enc_feat_dict[str(x.shape[-1])] = x.clone()
 
@@ -268,11 +266,11 @@ class CodeFormer(VQAutoEncoder):
         fuse_list = [self.fuse_generator_block[f_size] for f_size in self.connect_list]
 
         for i, block in enumerate(self.generator.blocks):
-            x = block(x) 
+            x = block(x)
             if i in fuse_list: # fuse after i-th block
                 f_size = str(x.shape[-1])
                 if w>0:
                     x = self.fuse_convs_dict[f_size](enc_feat_dict[f_size].detach(), x, w)
         out = x
         # logits doesn't need softmax before cross_entropy loss
-        return out, logits, lq_feat
+        return out, logits, lq_feat

modules/codeformer/vqgan_arch.py

@@ -5,17 +5,15 @@ VQGAN code, adapted from the original created by the Unleashing Transformers aut
 https://github.com/samb-t/unleashing-transformers/blob/master/models/vqgan.py
 
 '''
-import numpy as np
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-import copy
 from basicsr.utils import get_root_logger
 from basicsr.utils.registry import ARCH_REGISTRY
 
 def normalize(in_channels):
     return torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6, affine=True)
-    
+
 
 @torch.jit.script
 def swish(x):
@@ -212,15 +210,15 @@ class AttnBlock(nn.Module):
         # compute attention
         b, c, h, w = q.shape
         q = q.reshape(b, c, h*w)
-        q = q.permute(0, 2, 1)   
+        q = q.permute(0, 2, 1)
         k = k.reshape(b, c, h*w)
-        w_ = torch.bmm(q, k) 
+        w_ = torch.bmm(q, k)
         w_ = w_ * (int(c)**(-0.5))
         w_ = F.softmax(w_, dim=2)
 
         # attend to values
         v = v.reshape(b, c, h*w)
-        w_ = w_.permute(0, 2, 1) 
+        w_ = w_.permute(0, 2, 1)
         h_ = torch.bmm(v, w_)
         h_ = h_.reshape(b, c, h, w)
 
@@ -272,18 +270,18 @@ class Encoder(nn.Module):
     def forward(self, x):
         for block in self.blocks:
             x = block(x)
-            
+
         return x
 
 
 class Generator(nn.Module):
     def __init__(self, nf, emb_dim, ch_mult, res_blocks, img_size, attn_resolutions):
         super().__init__()
-        self.nf = nf 
-        self.ch_mult = ch_mult 
+        self.nf = nf
+        self.ch_mult = ch_mult
         self.num_resolutions = len(self.ch_mult)
         self.num_res_blocks = res_blocks
-        self.resolution = img_size 
+        self.resolution = img_size
         self.attn_resolutions = attn_resolutions
         self.in_channels = emb_dim
         self.out_channels = 3
@@ -317,29 +315,29 @@ class Generator(nn.Module):
         blocks.append(nn.Conv2d(block_in_ch, self.out_channels, kernel_size=3, stride=1, padding=1))
 
         self.blocks = nn.ModuleList(blocks)
-   
+
 
     def forward(self, x):
         for block in self.blocks:
             x = block(x)
-            
+
         return x
 
-  
+
 @ARCH_REGISTRY.register()
 class VQAutoEncoder(nn.Module):
-    def __init__(self, img_size, nf, ch_mult, quantizer="nearest", res_blocks=2, attn_resolutions=[16], codebook_size=1024, emb_dim=256,
+    def __init__(self, img_size, nf, ch_mult, quantizer="nearest", res_blocks=2, attn_resolutions=None, codebook_size=1024, emb_dim=256,
                 beta=0.25, gumbel_straight_through=False, gumbel_kl_weight=1e-8, model_path=None):
         super().__init__()
         logger = get_root_logger()
-        self.in_channels = 3 
-        self.nf = nf 
-        self.n_blocks = res_blocks 
+        self.in_channels = 3
+        self.nf = nf
+        self.n_blocks = res_blocks
         self.codebook_size = codebook_size
         self.embed_dim = emb_dim
         self.ch_mult = ch_mult
         self.resolution = img_size
-        self.attn_resolutions = attn_resolutions
+        self.attn_resolutions = attn_resolutions or [16]
         self.quantizer_type = quantizer
         self.encoder = Encoder(
             self.in_channels,
@@ -365,11 +363,11 @@ class VQAutoEncoder(nn.Module):
                 self.kl_weight
             )
         self.generator = Generator(
-            self.nf, 
+            self.nf,
             self.embed_dim,
-            self.ch_mult, 
-            self.n_blocks, 
-            self.resolution, 
+            self.ch_mult,
+            self.n_blocks,
+            self.resolution,
             self.attn_resolutions
         )
 
@@ -434,4 +432,4 @@ class VQGANDiscriminator(nn.Module):
                 raise ValueError('Wrong params!')
 
     def forward(self, x):
-        return self.main(x)
+        return self.main(x)

modules/codeformer_model.py

@@ -1,14 +1,12 @@
 import os
-import sys
-import traceback
 
 import cv2
 import torch
 
 import modules.face_restoration
 import modules.shared
-from modules import shared, devices, modelloader
-from modules.paths import script_path, models_path
+from modules import shared, devices, modelloader, errors
+from modules.paths import models_path
 
 # codeformer people made a choice to include modified basicsr library to their project which makes
 # it utterly impossible to use it alongside with other libraries that also use basicsr, like GFPGAN.
@@ -33,11 +31,9 @@ def setup_model(dirname):
     try:
         from torchvision.transforms.functional import normalize
         from modules.codeformer.codeformer_arch import CodeFormer
-        from basicsr.utils.download_util import load_file_from_url
-        from basicsr.utils import imwrite, img2tensor, tensor2img
+        from basicsr.utils import img2tensor, tensor2img
         from facelib.utils.face_restoration_helper import FaceRestoreHelper
         from facelib.detection.retinaface import retinaface
-        from modules.shared import cmd_opts
 
         net_class = CodeFormer
 
@@ -55,7 +51,7 @@ def setup_model(dirname):
                 if self.net is not None and self.face_helper is not None:
                     self.net.to(devices.device_codeformer)
                     return self.net, self.face_helper
-                model_paths = modelloader.load_models(model_path, model_url, self.cmd_dir, download_name='codeformer-v0.1.0.pth')
+                model_paths = modelloader.load_models(model_path, model_url, self.cmd_dir, download_name='codeformer-v0.1.0.pth', ext_filter=['.pth'])
                 if len(model_paths) != 0:
                     ckpt_path = model_paths[0]
                 else:
@@ -96,7 +92,7 @@ def setup_model(dirname):
                 self.face_helper.get_face_landmarks_5(only_center_face=False, resize=640, eye_dist_threshold=5)
                 self.face_helper.align_warp_face()
 
-                for idx, cropped_face in enumerate(self.face_helper.cropped_faces):
+                for cropped_face in self.face_helper.cropped_faces:
                     cropped_face_t = img2tensor(cropped_face / 255., bgr2rgb=True, float32=True)
                     normalize(cropped_face_t, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
                     cropped_face_t = cropped_face_t.unsqueeze(0).to(devices.device_codeformer)
@@ -107,8 +103,8 @@ def setup_model(dirname):
                             restored_face = tensor2img(output, rgb2bgr=True, min_max=(-1, 1))
                         del output
                         torch.cuda.empty_cache()
-                    except Exception as error:
-                        print(f'\tFailed inference for CodeFormer: {error}', file=sys.stderr)
+                    except Exception:
+                        errors.report('Failed inference for CodeFormer', exc_info=True)
                         restored_face = tensor2img(cropped_face_t, rgb2bgr=True, min_max=(-1, 1))
 
                     restored_face = restored_face.astype('uint8')
@@ -137,7 +133,6 @@ def setup_model(dirname):
         shared.face_restorers.append(codeformer)
 
     except Exception:
-        print("Error setting up CodeFormer:", file=sys.stderr)
-        print(traceback.format_exc(), file=sys.stderr)
+        errors.report("Error setting up CodeFormer", exc_info=True)
 
    # sys.path = stored_sys_path

modules/config_states.py

@@ -0,0 +1,197 @@
+"""
+Supports saving and restoring webui and extensions from a known working set of commits
+"""
+
+import os
+import json
+import time
+import tqdm
+
+from datetime import datetime
+from collections import OrderedDict
+import git
+
+from modules import shared, extensions, errors
+from modules.paths_internal import script_path, config_states_dir
+
+
+all_config_states = OrderedDict()
+
+
+def list_config_states():
+    global all_config_states
+
+    all_config_states.clear()
+    os.makedirs(config_states_dir, exist_ok=True)
+
+    config_states = []
+    for filename in os.listdir(config_states_dir):
+        if filename.endswith(".json"):
+            path = os.path.join(config_states_dir, filename)
+            with open(path, "r", encoding="utf-8") as f:
+                j = json.load(f)
+                j["filepath"] = path
+                config_states.append(j)
+
+    config_states = sorted(config_states, key=lambda cs: cs["created_at"], reverse=True)
+
+    for cs in config_states:
+        timestamp = time.asctime(time.gmtime(cs["created_at"]))
+        name = cs.get("name", "Config")
+        full_name = f"{name}: {timestamp}"
+        all_config_states[full_name] = cs
+
+    return all_config_states
+
+
+def get_webui_config():
+    webui_repo = None
+
+    try:
+        if os.path.exists(os.path.join(script_path, ".git")):
+            webui_repo = git.Repo(script_path)
+    except Exception:
+        errors.report(f"Error reading webui git info from {script_path}", exc_info=True)
+
+    webui_remote = None
+    webui_commit_hash = None
+    webui_commit_date = None
+    webui_branch = None
+    if webui_repo and not webui_repo.bare:
+        try:
+            webui_remote = next(webui_repo.remote().urls, None)
+            head = webui_repo.head.commit
+            webui_commit_date = webui_repo.head.commit.committed_date
+            webui_commit_hash = head.hexsha
+            webui_branch = webui_repo.active_branch.name
+
+        except Exception:
+            webui_remote = None
+
+    return {
+        "remote": webui_remote,
+        "commit_hash": webui_commit_hash,
+        "commit_date": webui_commit_date,
+        "branch": webui_branch,
+    }
+
+
+def get_extension_config():
+    ext_config = {}
+
+    for ext in extensions.extensions:
+        ext.read_info_from_repo()
+
+        entry = {
+            "name": ext.name,
+            "path": ext.path,
+            "enabled": ext.enabled,
+            "is_builtin": ext.is_builtin,
+            "remote": ext.remote,
+            "commit_hash": ext.commit_hash,
+            "commit_date": ext.commit_date,
+            "branch": ext.branch,
+            "have_info_from_repo": ext.have_info_from_repo
+        }
+
+        ext_config[ext.name] = entry
+
+    return ext_config
+
+
+def get_config():
+    creation_time = datetime.now().timestamp()
+    webui_config = get_webui_config()
+    ext_config = get_extension_config()
+
+    return {
+        "created_at": creation_time,
+        "webui": webui_config,
+        "extensions": ext_config
+    }
+
+
+def restore_webui_config(config):
+    print("* Restoring webui state...")
+
+    if "webui" not in config:
+        print("Error: No webui data saved to config")
+        return
+
+    webui_config = config["webui"]
+
+    if "commit_hash" not in webui_config:
+        print("Error: No commit saved to webui config")
+        return
+
+    webui_commit_hash = webui_config.get("commit_hash", None)
+    webui_repo = None
+
+    try:
+        if os.path.exists(os.path.join(script_path, ".git")):
+            webui_repo = git.Repo(script_path)
+    except Exception:
+        errors.report(f"Error reading webui git info from {script_path}", exc_info=True)
+        return
+
+    try:
+        webui_repo.git.fetch(all=True)
+        webui_repo.git.reset(webui_commit_hash, hard=True)
+        print(f"* Restored webui to commit {webui_commit_hash}.")
+    except Exception:
+        errors.report(f"Error restoring webui to commit{webui_commit_hash}")
+
+
+def restore_extension_config(config):
+    print("* Restoring extension state...")
+
+    if "extensions" not in config:
+        print("Error: No extension data saved to config")
+        return
+
+    ext_config = config["extensions"]
+
+    results = []
+    disabled = []
+
+    for ext in tqdm.tqdm(extensions.extensions):
+        if ext.is_builtin:
+            continue
+
+        ext.read_info_from_repo()
+        current_commit = ext.commit_hash
+
+        if ext.name not in ext_config:
+            ext.disabled = True
+            disabled.append(ext.name)
+            results.append((ext, current_commit[:8], False, "Saved extension state not found in config, marking as disabled"))
+            continue
+
+        entry = ext_config[ext.name]
+
+        if "commit_hash" in entry and entry["commit_hash"]:
+            try:
+                ext.fetch_and_reset_hard(entry["commit_hash"])
+                ext.read_info_from_repo()
+                if current_commit != entry["commit_hash"]:
+                    results.append((ext, current_commit[:8], True, entry["commit_hash"][:8]))
+            except Exception as ex:
+                results.append((ext, current_commit[:8], False, ex))
+        else:
+            results.append((ext, current_commit[:8], False, "No commit hash found in config"))
+
+        if not entry.get("enabled", False):
+            ext.disabled = True
+            disabled.append(ext.name)
+        else:
+            ext.disabled = False
+
+    shared.opts.disabled_extensions = disabled
+    shared.opts.save(shared.config_filename)
+
+    print("* Finished restoring extensions. Results:")
+    for ext, prev_commit, success, result in results:
+        if success:
+            print(f"  + {ext.name}: {prev_commit} -> {result}")
+        else:
+            print(f"  ! {ext.name}: FAILURE ({result})")

modules/deepbooru.py

@@ -2,7 +2,6 @@ import os
 import re
 
 import torch
-from PIL import Image
 import numpy as np
 
 from modules import modelloader, paths, deepbooru_model, devices, images, shared
@@ -79,7 +78,7 @@ class DeepDanbooru:
 
         res = []
 
-        filtertags = set([x.strip().replace(' ', '_') for x in shared.opts.deepbooru_filter_tags.split(",")])
+        filtertags = {x.strip().replace(' ', '_') for x in shared.opts.deepbooru_filter_tags.split(",")}
 
         for tag in [x for x in tags if x not in filtertags]:
             probability = probability_dict[tag]

modules/deepbooru_model.py

@@ -2,6 +2,8 @@ import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
+from modules import devices
+
 # see https://github.com/AUTOMATIC1111/TorchDeepDanbooru for more
 
 
@@ -196,7 +198,7 @@ class DeepDanbooruModel(nn.Module):
         t_358, = inputs
         t_359 = t_358.permute(*[0, 3, 1, 2])
         t_359_padded = F.pad(t_359, [2, 3, 2, 3], value=0)
-        t_360 = self.n_Conv_0(t_359_padded)
+        t_360 = self.n_Conv_0(t_359_padded.to(self.n_Conv_0.bias.dtype) if devices.unet_needs_upcast else t_359_padded)
         t_361 = F.relu(t_360)
         t_361 = F.pad(t_361, [0, 1, 0, 1], value=float('-inf'))
         t_362 = self.n_MaxPool_0(t_361)

modules/devices.py

@@ -1,21 +1,19 @@
-import sys, os, shlex
+import sys
 import contextlib
+from functools import lru_cache
+
 import torch
 from modules import errors
-from packaging import version
+
+if sys.platform == "darwin":
+    from modules import mac_specific
 
 
-# has_mps is only available in nightly pytorch (for now) and macOS 12.3+.
-# check `getattr` and try it for compatibility
 def has_mps() -> bool:
-    if not getattr(torch, 'has_mps', False):
+    if sys.platform != "darwin":
         return False
-    try:
-        torch.zeros(1).to(torch.device("mps"))
-        return True
-    except Exception:
-        return False
-
+    else:
+        return mac_specific.has_mps
 
 def extract_device_id(args, name):
     for x in range(len(args)):
@@ -34,14 +32,18 @@ def get_cuda_device_string():
     return "cuda"
 
 
-def get_optimal_device():
+def get_optimal_device_name():
     if torch.cuda.is_available():
-        return torch.device(get_cuda_device_string())
+        return get_cuda_device_string()
 
     if has_mps():
-        return torch.device("mps")
+        return "mps"
 
-    return cpu
+    return "cpu"
+
+
+def get_optimal_device():
+    return torch.device(get_optimal_device_name())
 
 
 def get_device_for(task):
@@ -65,7 +67,7 @@ def enable_tf32():
 
         # enabling benchmark option seems to enable a range of cards to do fp16 when they otherwise can't
         # see https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/4407
-        if any([torch.cuda.get_device_capability(devid) == (7, 5) for devid in range(0, torch.cuda.device_count())]):
+        if any(torch.cuda.get_device_capability(devid) == (7, 5) for devid in range(0, torch.cuda.device_count())):
             torch.backends.cudnn.benchmark = True
 
         torch.backends.cuda.matmul.allow_tf32 = True
@@ -79,17 +81,31 @@ cpu = torch.device("cpu")
 device = device_interrogate = device_gfpgan = device_esrgan = device_codeformer = None
 dtype = torch.float16
 dtype_vae = torch.float16
+dtype_unet = torch.float16
+unet_needs_upcast = False
+
+
+def cond_cast_unet(input):
+    return input.to(dtype_unet) if unet_needs_upcast else input
+
+
+def cond_cast_float(input):
+    return input.float() if unet_needs_upcast else input
 
 
 def randn(seed, shape):
+    from modules.shared import opts
+
     torch.manual_seed(seed)
-    if device.type == 'mps':
+    if opts.randn_source == "CPU" or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
 
 def randn_without_seed(shape):
-    if device.type == 'mps':
+    from modules.shared import opts
+
+    if opts.randn_source == "CPU" or device.type == 'mps':
         return torch.randn(shape, device=cpu).to(device)
     return torch.randn(shape, device=device)
 
@@ -106,6 +122,10 @@ def autocast(disable=False):
     return torch.autocast("cuda")
 
 
+def without_autocast(disable=False):
+    return torch.autocast("cuda", enabled=False) if torch.is_autocast_enabled() and not disable else contextlib.nullcontext()
+
+
 class NansException(Exception):
     pass
 
@@ -123,7 +143,7 @@ def test_for_nans(x, where):
         message = "A tensor with all NaNs was produced in Unet."
 
         if not shared.cmd_opts.no_half:
-            message += " This could be either because there's not enough precision to represent the picture, or because your video card does not support half type. Try using --no-half commandline argument to fix this."
+            message += " This could be either because there's not enough precision to represent the picture, or because your video card does not support half type. Try setting the \"Upcast cross attention layer to float32\" option in Settings > Stable Diffusion or using the --no-half commandline argument to fix this."
 
     elif where == "vae":
         message = "A tensor with all NaNs was produced in VAE."
@@ -133,60 +153,22 @@ def test_for_nans(x, where):
     else:
         message = "A tensor with all NaNs was produced."
 
+    message += " Use --disable-nan-check commandline argument to disable this check."
+
     raise NansException(message)
 
 
-# MPS workaround for https://github.com/pytorch/pytorch/issues/79383
-orig_tensor_to = torch.Tensor.to
-def tensor_to_fix(self, *args, **kwargs):
-    if self.device.type != 'mps' and \
-       ((len(args) > 0 and isinstance(args[0], torch.device) and args[0].type == 'mps') or \
-       (isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')):
-        self = self.contiguous()
-    return orig_tensor_to(self, *args, **kwargs)
+@lru_cache
+def first_time_calculation():
+    """
+    just do any calculation with pytorch layers - the first time this is done it allocaltes about 700MB of memory and
+    spends about 2.7 seconds doing that, at least wih NVidia.
+    """
 
+    x = torch.zeros((1, 1)).to(device, dtype)
+    linear = torch.nn.Linear(1, 1).to(device, dtype)
+    linear(x)
 
-# MPS workaround for https://github.com/pytorch/pytorch/issues/80800 
-orig_layer_norm = torch.nn.functional.layer_norm
-def layer_norm_fix(*args, **kwargs):
-    if len(args) > 0 and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps':
-        args = list(args)
-        args[0] = args[0].contiguous()
-    return orig_layer_norm(*args, **kwargs)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/90532
-orig_tensor_numpy = torch.Tensor.numpy
-def numpy_fix(self, *args, **kwargs):
-    if self.requires_grad:
-        self = self.detach()
-    return orig_tensor_numpy(self, *args, **kwargs)
-
-
-# MPS workaround for https://github.com/pytorch/pytorch/issues/89784
-orig_cumsum = torch.cumsum
-orig_Tensor_cumsum = torch.Tensor.cumsum
-def cumsum_fix(input, cumsum_func, *args, **kwargs):
-    if input.device.type == 'mps':
-        output_dtype = kwargs.get('dtype', input.dtype)
-        if output_dtype == torch.int64:
-            return cumsum_func(input.cpu(), *args, **kwargs).to(input.device)
-        elif cumsum_needs_bool_fix and output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16):
-            return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64)
-    return cumsum_func(input, *args, **kwargs)
-
-
-if has_mps():
-    if version.parse(torch.__version__) < version.parse("1.13"):
-        # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working
-        torch.Tensor.to = tensor_to_fix
-        torch.nn.functional.layer_norm = layer_norm_fix
-        torch.Tensor.numpy = numpy_fix
-    elif version.parse(torch.__version__) > version.parse("1.13.1"):
-        cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0))
-        cumsum_needs_bool_fix = not torch.BoolTensor([True,True]).to(device=torch.device("mps"), dtype=torch.int64).equal(torch.BoolTensor([True,False]).to(torch.device("mps")).cumsum(0))
-        torch.cumsum = lambda input, *args, **kwargs: ( cumsum_fix(input, orig_cumsum, *args, **kwargs) )
-        torch.Tensor.cumsum = lambda self, *args, **kwargs: ( cumsum_fix(self, orig_Tensor_cumsum, *args, **kwargs) )
-        orig_narrow = torch.narrow
-        torch.narrow = lambda *args, **kwargs: ( orig_narrow(*args, **kwargs).clone() )
-
+    x = torch.zeros((1, 1, 3, 3)).to(device, dtype)
+    conv2d = torch.nn.Conv2d(1, 1, (3, 3)).to(device, dtype)
+    conv2d(x)

modules/errors.py

@@ -1,8 +1,42 @@
 import sys
+import textwrap
 import traceback
 
 
+exception_records = []
+
+
+def record_exception():
+    _, e, tb = sys.exc_info()
+    if e is None:
+        return
+
+    if exception_records and exception_records[-1] == e:
+        return
+
+    exception_records.append((e, tb))
+
+    if len(exception_records) > 5:
+        exception_records.pop(0)
+
+
+def report(message: str, *, exc_info: bool = False) -> None:
+    """
+    Print an error message to stderr, with optional traceback.
+    """
+
+    record_exception()
+
+    for line in message.splitlines():
+        print("***", line, file=sys.stderr)
+    if exc_info:
+        print(textwrap.indent(traceback.format_exc(), "    "), file=sys.stderr)
+        print("---", file=sys.stderr)
+
+
 def print_error_explanation(message):
+    record_exception()
+
     lines = message.strip().split("\n")
     max_len = max([len(x) for x in lines])
 
@@ -12,9 +46,15 @@ def print_error_explanation(message):
     print('=' * max_len, file=sys.stderr)
 
 
-def display(e: Exception, task):
+def display(e: Exception, task, *, full_traceback=False):
+    record_exception()
+
     print(f"{task or 'error'}: {type(e).__name__}", file=sys.stderr)
-    print(traceback.format_exc(), file=sys.stderr)
+    te = traceback.TracebackException.from_exception(e)
+    if full_traceback:
+        # include frames leading up to the try-catch block
+        te.stack = traceback.StackSummary(traceback.extract_stack()[:-2] + te.stack)
+    print(*te.format(), sep="", file=sys.stderr)
 
     message = str(e)
     if "copying a param with shape torch.Size([640, 1024]) from checkpoint, the shape in current model is torch.Size([640, 768])" in message:
@@ -28,6 +68,8 @@ already_displayed = {}
 
 
 def display_once(e: Exception, task):
+    record_exception()
+
     if task in already_displayed:
         return
 

modules/esrgan_model.py

@@ -6,7 +6,7 @@ from PIL import Image
 from basicsr.utils.download_util import load_file_from_url
 
 import modules.esrgan_model_arch as arch
-from modules import shared, modelloader, images, devices
+from modules import modelloader, images, devices
 from modules.upscaler import Upscaler, UpscalerData
 from modules.shared import opts
 
@@ -16,9 +16,7 @@ def mod2normal(state_dict):
     # this code is copied from https://github.com/victorca25/iNNfer
     if 'conv_first.weight' in state_dict:
         crt_net = {}
-        items = []
-        for k, v in state_dict.items():
-            items.append(k)
+        items = list(state_dict)
 
         crt_net['model.0.weight'] = state_dict['conv_first.weight']
         crt_net['model.0.bias'] = state_dict['conv_first.bias']
@@ -52,9 +50,7 @@ def resrgan2normal(state_dict, nb=23):
     if "conv_first.weight" in state_dict and "body.0.rdb1.conv1.weight" in state_dict:
         re8x = 0
         crt_net = {}
-        items = []
-        for k, v in state_dict.items():
-            items.append(k)
+        items = list(state_dict)
 
         crt_net['model.0.weight'] = state_dict['conv_first.weight']
         crt_net['model.0.bias'] = state_dict['conv_first.bias']
@@ -156,13 +152,16 @@ class UpscalerESRGAN(Upscaler):
 
     def load_model(self, path: str):
         if "http" in path:
-            filename = load_file_from_url(url=self.model_url, model_dir=self.model_path,
-                                          file_name="%s.pth" % self.model_name,
-                                          progress=True)
+            filename = load_file_from_url(
+                url=self.model_url,
+                model_dir=self.model_download_path,
+                file_name=f"{self.model_name}.pth",
+                progress=True,
+            )
         else:
             filename = path
         if not os.path.exists(filename) or filename is None:
-            print("Unable to load %s from %s" % (self.model_path, filename))
+            print(f"Unable to load {self.model_path} from {filename}")
             return None
 
         state_dict = torch.load(filename, map_location='cpu' if devices.device_esrgan.type == 'mps' else None)

modules/esrgan_model_arch.py

@@ -1,7 +1,7 @@
 # this file is adapted from https://github.com/victorca25/iNNfer
 
+from collections import OrderedDict
 import math
-import functools
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -37,7 +37,7 @@ class RRDBNet(nn.Module):
         elif upsample_mode == 'pixelshuffle':
             upsample_block = pixelshuffle_block
         else:
-            raise NotImplementedError('upsample mode [{:s}] is not found'.format(upsample_mode))
+            raise NotImplementedError(f'upsample mode [{upsample_mode}] is not found')
         if upscale == 3:
             upsampler = upsample_block(nf, nf, 3, act_type=act_type, convtype=convtype)
         else:
@@ -105,7 +105,7 @@ class ResidualDenseBlock_5C(nn.Module):
     Modified options that can be used:
         - "Partial Convolution based Padding" arXiv:1811.11718
         - "Spectral normalization" arXiv:1802.05957
-        - "ICASSP 2020 - ESRGAN+ : Further Improving ESRGAN" N. C. 
+        - "ICASSP 2020 - ESRGAN+ : Further Improving ESRGAN" N. C.
             {Rakotonirina} and A. {Rasoanaivo}
     """
 
@@ -170,7 +170,7 @@ class GaussianNoise(nn.Module):
             scale = self.sigma * x.detach() if self.is_relative_detach else self.sigma * x
             sampled_noise = self.noise.repeat(*x.size()).normal_() * scale
             x = x + sampled_noise
-        return x 
+        return x
 
 def conv1x1(in_planes, out_planes, stride=1):
     return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
@@ -260,10 +260,10 @@ class Upsample(nn.Module):
 
     def extra_repr(self):
         if self.scale_factor is not None:
-            info = 'scale_factor=' + str(self.scale_factor)
+            info = f'scale_factor={self.scale_factor}'
         else:
-            info = 'size=' + str(self.size)
-        info += ', mode=' + self.mode
+            info = f'size={self.size}'
+        info += f', mode={self.mode}'
         return info
 
 
@@ -349,7 +349,7 @@ def act(act_type, inplace=True, neg_slope=0.2, n_prelu=1, beta=1.0):
     elif act_type == 'sigmoid':  # [0, 1] range output
         layer = nn.Sigmoid()
     else:
-        raise NotImplementedError('activation layer [{:s}] is not found'.format(act_type))
+        raise NotImplementedError(f'activation layer [{act_type}] is not found')
     return layer
 
 
@@ -371,7 +371,7 @@ def norm(norm_type, nc):
     elif norm_type == 'none':
         def norm_layer(x): return Identity()
     else:
-        raise NotImplementedError('normalization layer [{:s}] is not found'.format(norm_type))
+        raise NotImplementedError(f'normalization layer [{norm_type}] is not found')
     return layer
 
 
@@ -387,7 +387,7 @@ def pad(pad_type, padding):
     elif pad_type == 'zero':
         layer = nn.ZeroPad2d(padding)
     else:
-        raise NotImplementedError('padding layer [{:s}] is not implemented'.format(pad_type))
+        raise NotImplementedError(f'padding layer [{pad_type}] is not implemented')
     return layer
 
 
@@ -431,15 +431,17 @@ def conv_block(in_nc, out_nc, kernel_size, stride=1, dilation=1, groups=1, bias=
                pad_type='zero', norm_type=None, act_type='relu', mode='CNA', convtype='Conv2D',
                spectral_norm=False):
     """ Conv layer with padding, normalization, activation """
-    assert mode in ['CNA', 'NAC', 'CNAC'], 'Wrong conv mode [{:s}]'.format(mode)
+    assert mode in ['CNA', 'NAC', 'CNAC'], f'Wrong conv mode [{mode}]'
     padding = get_valid_padding(kernel_size, dilation)
     p = pad(pad_type, padding) if pad_type and pad_type != 'zero' else None
     padding = padding if pad_type == 'zero' else 0
 
     if convtype=='PartialConv2D':
+        from torchvision.ops import PartialConv2d  # this is definitely not going to work, but PartialConv2d doesn't work anyway and this shuts up static analyzer
         c = PartialConv2d(in_nc, out_nc, kernel_size=kernel_size, stride=stride, padding=padding,
                dilation=dilation, bias=bias, groups=groups)
     elif convtype=='DeformConv2D':
+        from torchvision.ops import DeformConv2d  # not tested
         c = DeformConv2d(in_nc, out_nc, kernel_size=kernel_size, stride=stride, padding=padding,
                dilation=dilation, bias=bias, groups=groups)
     elif convtype=='Conv3D':

modules/extensions.py

@@ -1,21 +1,29 @@
 import os
-import sys
-import traceback
+import threading
 
-import git
-
-from modules import paths, shared
+from modules import shared, errors
+# from modules.gitpython_hack import Repo
+from git import Repo
+from modules.paths_internal import extensions_dir, extensions_builtin_dir, script_path  # noqa: F401
 
 extensions = []
-extensions_dir = os.path.join(paths.script_path, "extensions")
-extensions_builtin_dir = os.path.join(paths.script_path, "extensions-builtin")
+
+if not os.path.exists(extensions_dir):
+    os.makedirs(extensions_dir)
 
 
 def active():
-    return [x for x in extensions if x.enabled]
+    if shared.opts.disable_all_extensions == "all":
+        return []
+    elif shared.opts.disable_all_extensions == "extra":
+        return [x for x in extensions if x.enabled and x.is_builtin]
+    else:
+        return [x for x in extensions if x.enabled]
 
 
 class Extension:
+    lock = threading.Lock()
+
     def __init__(self, name, path, enabled=True, is_builtin=False):
         self.name = name
         self.path = path
@@ -23,24 +31,50 @@ class Extension:
         self.status = ''
         self.can_update = False
         self.is_builtin = is_builtin
+        self.commit_hash = ''
+        self.commit_date = None
+        self.version = ''
+        self.branch = None
+        self.remote = None
+        self.have_info_from_repo = False
 
+    def read_info_from_repo(self):
+        if self.is_builtin or self.have_info_from_repo:
+            return
+
+        with self.lock:
+            if self.have_info_from_repo:
+                return
+
+            self.do_read_info_from_repo()
+
+    def do_read_info_from_repo(self):
         repo = None
         try:
-            if os.path.exists(os.path.join(path, ".git")):
-                repo = git.Repo(path)
+            if os.path.exists(os.path.join(self.path, ".git")):
+                repo = Repo(self.path)
         except Exception:
-            print(f"Error reading github repository info from {path}:", file=sys.stderr)
-            print(traceback.format_exc(), file=sys.stderr)
+            errors.report(f"Error reading github repository info from {self.path}", exc_info=True)
 
         if repo is None or repo.bare:
             self.remote = None
         else:
             try:
-                self.remote = next(repo.remote().urls, None)
                 self.status = 'unknown'
+                self.remote = next(repo.remote().urls, None)
+                commit = repo.head.commit
+                self.commit_date = commit.committed_date
+                if repo.active_branch:
+                    self.branch = repo.active_branch.name
+                self.commit_hash = commit.hexsha
+                self.version = self.commit_hash[:8]
+
             except Exception:
+                errors.report(f"Failed reading extension data from Git repository ({self.name})", exc_info=True)
                 self.remote = None
 
+        self.have_info_from_repo = True
+
     def list_files(self, subdir, extension):
         from modules import scripts
 
@@ -57,22 +91,34 @@ class Extension:
         return res
 
     def check_updates(self):
-        repo = git.Repo(self.path)
-        for fetch in repo.remote().fetch("--dry-run"):
+        repo = Repo(self.path)
+        for fetch in repo.remote().fetch(dry_run=True):
             if fetch.flags != fetch.HEAD_UPTODATE:
                 self.can_update = True
-                self.status = "behind"
+                self.status = "new commits"
                 return
 
+        try:
+            origin = repo.rev_parse('origin')
+            if repo.head.commit != origin:
+                self.can_update = True
+                self.status = "behind HEAD"
+                return
+        except Exception:
+            self.can_update = False
+            self.status = "unknown (remote error)"
+            return
+
         self.can_update = False
         self.status = "latest"
 
-    def fetch_and_reset_hard(self):
-        repo = git.Repo(self.path)
+    def fetch_and_reset_hard(self, commit='origin'):
+        repo = Repo(self.path)
         # Fix: `error: Your local changes to the following files would be overwritten by merge`,
         # because WSL2 Docker set 755 file permissions instead of 644, this results to the error.
-        repo.git.fetch('--all')
-        repo.git.reset('--hard', 'origin')
+        repo.git.fetch(all=True)
+        repo.git.reset(commit, hard=True)
+        self.have_info_from_repo = False
 
 
 def list_extensions():
@@ -81,7 +127,12 @@ def list_extensions():
     if not os.path.isdir(extensions_dir):
         return
 
-    paths = []
+    if shared.opts.disable_all_extensions == "all":
+        print("*** \"Disable all extensions\" option was set, will not load any extensions ***")
+    elif shared.opts.disable_all_extensions == "extra":
+        print("*** \"Disable all extensions\" option was set, will only load built-in extensions ***")
+
+    extension_paths = []
     for dirname in [extensions_dir, extensions_builtin_dir]:
         if not os.path.isdir(dirname):
             return
@@ -91,9 +142,8 @@ def list_extensions():
             if not os.path.isdir(path):
                 continue
 
-            paths.append((extension_dirname, path, dirname == extensions_builtin_dir))
+            extension_paths.append((extension_dirname, path, dirname == extensions_builtin_dir))
 
-    for dirname, path, is_builtin in paths:
+    for dirname, path, is_builtin in extension_paths:
         extension = Extension(name=dirname, path=path, enabled=dirname not in shared.opts.disabled_extensions, is_builtin=is_builtin)
         extensions.append(extension)
-

modules/extra_networks.py

@@ -14,9 +14,26 @@ def register_extra_network(extra_network):
     extra_network_registry[extra_network.name] = extra_network
 
 
+def register_default_extra_networks():
+    from modules.extra_networks_hypernet import ExtraNetworkHypernet
+    register_extra_network(ExtraNetworkHypernet())
+
+
 class ExtraNetworkParams:
     def __init__(self, items=None):
         self.items = items or []
+        self.positional = []
+        self.named = {}
+
+        for item in self.items:
+            parts = item.split('=', 2) if isinstance(item, str) else [item]
+            if len(parts) == 2:
+                self.named[parts[0]] = parts[1]
+            else:
+                self.positional.append(item)
+
+    def __eq__(self, other):
+        return self.items == other.items
 
 
 class ExtraNetwork:
@@ -91,7 +108,7 @@ def deactivate(p, extra_network_data):
     """call deactivate for extra networks in extra_network_data in specified order, then call
     deactivate for all remaining registered networks"""
 
-    for extra_network_name, extra_network_args in extra_network_data.items():
+    for extra_network_name in extra_network_data:
         extra_network = extra_network_registry.get(extra_network_name, None)
         if extra_network is None:
             continue

modules/extra_networks_hypernet.py

@@ -1,4 +1,4 @@
-from modules import extra_networks
+from modules import extra_networks, shared
 from modules.hypernetworks import hypernetwork
 
 
@@ -7,10 +7,17 @@ class ExtraNetworkHypernet(extra_networks.ExtraNetwork):
         super().__init__('hypernet')
 
     def activate(self, p, params_list):
+        additional = shared.opts.sd_hypernetwork
+
+        if additional != "None" and additional in shared.hypernetworks and not any(x for x in params_list if x.items[0] == additional):
+            hypernet_prompt_text = f"<hypernet:{additional}:{shared.opts.extra_networks_default_multiplier}>"
+            p.all_prompts = [f"{prompt}{hypernet_prompt_text}" for prompt in p.all_prompts]
+            params_list.append(extra_networks.ExtraNetworkParams(items=[additional, shared.opts.extra_networks_default_multiplier]))
+
         names = []
         multipliers = []
         for params in params_list:
-            assert len(params.items) > 0
+            assert params.items
 
             names.append(params.items[0])
             multipliers.append(float(params.items[1]) if len(params.items) > 1 else 1.0)

modules/extras.py

@@ -1,12 +1,13 @@
 import os
 import re
 import shutil
+import json
 
 
 import torch
 import tqdm
 
-from modules import shared, images, sd_models, sd_vae
+from modules import shared, images, sd_models, sd_vae, sd_models_config
 from modules.ui_common import plaintext_to_html
 import gradio as gr
 import safetensors.torch
@@ -37,7 +38,7 @@ def run_pnginfo(image):
 
 def create_config(ckpt_result, config_source, a, b, c):
     def config(x):
-        res = sd_models.find_checkpoint_config(x) if x else None
+        res = sd_models_config.find_checkpoint_config_near_filename(x) if x else None
         return res if res != shared.sd_default_config else None
 
     if config_source == 0:
@@ -71,7 +72,7 @@ def to_half(tensor, enable):
     return tensor
 
 
-def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_model_name, interp_method, multiplier, save_as_half, custom_name, checkpoint_format, config_source, bake_in_vae, discard_weights):
+def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_model_name, interp_method, multiplier, save_as_half, custom_name, checkpoint_format, config_source, bake_in_vae, discard_weights, save_metadata):
     shared.state.begin()
     shared.state.job = 'model-merge'
 
@@ -132,16 +133,17 @@ def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_
     tertiary_model_info = sd_models.checkpoints_list[tertiary_model_name] if theta_func1 else None
 
     result_is_inpainting_model = False
+    result_is_instruct_pix2pix_model = False
 
     if theta_func2:
-        shared.state.textinfo = f"Loading B"
+        shared.state.textinfo = "Loading B"
         print(f"Loading {secondary_model_info.filename}...")
         theta_1 = sd_models.read_state_dict(secondary_model_info.filename, map_location='cpu')
     else:
         theta_1 = None
 
     if theta_func1:
-        shared.state.textinfo = f"Loading C"
+        shared.state.textinfo = "Loading C"
         print(f"Loading {tertiary_model_info.filename}...")
         theta_2 = sd_models.read_state_dict(tertiary_model_info.filename, map_location='cpu')
 
@@ -185,11 +187,16 @@ def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_
             if a.shape != b.shape and a.shape[0:1] + a.shape[2:] == b.shape[0:1] + b.shape[2:]:
                 if a.shape[1] == 4 and b.shape[1] == 9:
                     raise RuntimeError("When merging inpainting model with a normal one, A must be the inpainting model.")
+                if a.shape[1] == 4 and b.shape[1] == 8:
+                    raise RuntimeError("When merging instruct-pix2pix model with a normal one, A must be the instruct-pix2pix model.")
 
-                assert a.shape[1] == 9 and b.shape[1] == 4, f"Bad dimensions for merged layer {key}: A={a.shape}, B={b.shape}"
-
-                theta_0[key][:, 0:4, :, :] = theta_func2(a[:, 0:4, :, :], b, multiplier)
-                result_is_inpainting_model = True
+                if a.shape[1] == 8 and b.shape[1] == 4:#If we have an Instruct-Pix2Pix model...
+                    theta_0[key][:, 0:4, :, :] = theta_func2(a[:, 0:4, :, :], b, multiplier)#Merge only the vectors the models have in common.  Otherwise we get an error due to dimension mismatch.
+                    result_is_instruct_pix2pix_model = True
+                else:
+                    assert a.shape[1] == 9 and b.shape[1] == 4, f"Bad dimensions for merged layer {key}: A={a.shape}, B={b.shape}"
+                    theta_0[key][:, 0:4, :, :] = theta_func2(a[:, 0:4, :, :], b, multiplier)
+                    result_is_inpainting_model = True
             else:
                 theta_0[key] = theta_func2(a, b, multiplier)
 
@@ -226,6 +233,7 @@ def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_
 
     filename = filename_generator() if custom_name == '' else custom_name
     filename += ".inpainting" if result_is_inpainting_model else ""
+    filename += ".instruct-pix2pix" if result_is_instruct_pix2pix_model else ""
     filename += "." + checkpoint_format
 
     output_modelname = os.path.join(ckpt_dir, filename)
@@ -234,13 +242,58 @@ def run_modelmerger(id_task, primary_model_name, secondary_model_name, tertiary_
     shared.state.textinfo = "Saving"
     print(f"Saving to {output_modelname}...")
 
+    metadata = None
+
+    if save_metadata:
+        metadata = {"format": "pt"}
+
+        merge_recipe = {
+            "type": "webui", # indicate this model was merged with webui's built-in merger
+            "primary_model_hash": primary_model_info.sha256,
+            "secondary_model_hash": secondary_model_info.sha256 if secondary_model_info else None,
+            "tertiary_model_hash": tertiary_model_info.sha256 if tertiary_model_info else None,
+            "interp_method": interp_method,
+            "multiplier": multiplier,
+            "save_as_half": save_as_half,
+            "custom_name": custom_name,
+            "config_source": config_source,
+            "bake_in_vae": bake_in_vae,
+            "discard_weights": discard_weights,
+            "is_inpainting": result_is_inpainting_model,
+            "is_instruct_pix2pix": result_is_instruct_pix2pix_model
+        }
+        metadata["sd_merge_recipe"] = json.dumps(merge_recipe)
+
+        sd_merge_models = {}
+
+        def add_model_metadata(checkpoint_info):
+            checkpoint_info.calculate_shorthash()
+            sd_merge_models[checkpoint_info.sha256] = {
+                "name": checkpoint_info.name,
+                "legacy_hash": checkpoint_info.hash,
+                "sd_merge_recipe": checkpoint_info.metadata.get("sd_merge_recipe", None)
+            }
+
+            sd_merge_models.update(checkpoint_info.metadata.get("sd_merge_models", {}))
+
+        add_model_metadata(primary_model_info)
+        if secondary_model_info:
+            add_model_metadata(secondary_model_info)
+        if tertiary_model_info:
+            add_model_metadata(tertiary_model_info)
+
+        metadata["sd_merge_models"] = json.dumps(sd_merge_models)
+
     _, extension = os.path.splitext(output_modelname)
     if extension.lower() == ".safetensors":
-        safetensors.torch.save_file(theta_0, output_modelname, metadata={"format": "pt"})
+        safetensors.torch.save_file(theta_0, output_modelname, metadata=metadata)
     else:
         torch.save(theta_0, output_modelname)
 
     sd_models.list_models()
+    created_model = next((ckpt for ckpt in sd_models.checkpoints_list.values() if ckpt.name == filename), None)
+    if created_model:
+        created_model.calculate_shorthash()
 
     create_config(output_modelname, config_source, primary_model_info, secondary_model_info, tertiary_model_info)
 

modules/generation_parameters_copypaste.py

@@ -1,46 +1,61 @@
 import base64
 import io
-import math
+import json
 import os
 import re
-from pathlib import Path
 
 import gradio as gr
-from modules.shared import script_path
+from modules.paths import data_path
 from modules import shared, ui_tempdir, script_callbacks
-import tempfile
 from PIL import Image
 
-re_param_code = r'\s*([\w ]+):\s*("(?:\\|\"|[^\"])+"|[^,]*)(?:,|$)'
+re_param_code = r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)'
 re_param = re.compile(re_param_code)
-re_params = re.compile(r"^(?:" + re_param_code + "){3,}$")
 re_imagesize = re.compile(r"^(\d+)x(\d+)$")
 re_hypernet_hash = re.compile("\(([0-9a-f]+)\)$")
 type_of_gr_update = type(gr.update())
+
 paste_fields = {}
-bind_list = []
+registered_param_bindings = []
+
+
+class ParamBinding:
+    def __init__(self, paste_button, tabname, source_text_component=None, source_image_component=None, source_tabname=None, override_settings_component=None, paste_field_names=None):
+        self.paste_button = paste_button
+        self.tabname = tabname
+        self.source_text_component = source_text_component
+        self.source_image_component = source_image_component
+        self.source_tabname = source_tabname
+        self.override_settings_component = override_settings_component
+        self.paste_field_names = paste_field_names or []
 
 
 def reset():
     paste_fields.clear()
-    bind_list.clear()
 
 
 def quote(text):
-    if ',' not in str(text):
+    if ',' not in str(text) and '\n' not in str(text) and ':' not in str(text):
         return text
 
-    text = str(text)
-    text = text.replace('\\', '\\\\')
-    text = text.replace('"', '\\"')
-    return f'"{text}"'
+    return json.dumps(text, ensure_ascii=False)
+
+
+def unquote(text):
+    if len(text) == 0 or text[0] != '"' or text[-1] != '"':
+        return text
+
+    try:
+        return json.loads(text)
+    except Exception:
+        return text
 
 
 def image_from_url_text(filedata):
     if filedata is None:
         return None
 
-    if type(filedata) == list and len(filedata) > 0 and type(filedata[0]) == dict and filedata[0].get("is_file", False):
+    if type(filedata) == list and filedata and type(filedata[0]) == dict and filedata[0].get("is_file", False):
         filedata = filedata[0]
 
     if type(filedata) == dict and filedata.get("is_file", False):
@@ -48,6 +63,7 @@ def image_from_url_text(filedata):
         is_in_right_dir = ui_tempdir.check_tmp_file(shared.demo, filename)
         assert is_in_right_dir, 'trying to open image file outside of allowed directories'
 
+        filename = filename.rsplit('?', 1)[0]
         return Image.open(filename)
 
     if type(filedata) == list:
@@ -64,8 +80,8 @@ def image_from_url_text(filedata):
     return image
 
 
-def add_paste_fields(tabname, init_img, fields):
-    paste_fields[tabname] = {"init_img": init_img, "fields": fields}
+def add_paste_fields(tabname, init_img, fields, override_settings_component=None):
+    paste_fields[tabname] = {"init_img": init_img, "fields": fields, "override_settings_component": override_settings_component}
 
     # backwards compatibility for existing extensions
     import modules.ui
@@ -75,26 +91,6 @@ def add_paste_fields(tabname, init_img, fields):
         modules.ui.img2img_paste_fields = fields
 
 
-def integrate_settings_paste_fields(component_dict):
-    from modules import ui
-
-    settings_map = {
-        'CLIP_stop_at_last_layers': 'Clip skip',
-        'inpainting_mask_weight': 'Conditional mask weight',
-        'sd_model_checkpoint': 'Model hash',
-        'eta_noise_seed_delta': 'ENSD',
-        'initial_noise_multiplier': 'Noise multiplier',
-    }
-    settings_paste_fields = [
-        (component_dict[k], lambda d, k=k, v=v: ui.apply_setting(k, d.get(v, None)))
-        for k, v in settings_map.items()
-    ]
-
-    for tabname, info in paste_fields.items():
-        if info["fields"] is not None:
-            info["fields"] += settings_paste_fields
-
-
 def create_buttons(tabs_list):
     buttons = {}
     for tab in tabs_list:
@@ -102,9 +98,64 @@ def create_buttons(tabs_list):
     return buttons
 
 
-#if send_generate_info is a tab name, mean generate_info comes from the params fields of the tab
 def bind_buttons(buttons, send_image, send_generate_info):
-    bind_list.append([buttons, send_image, send_generate_info])
+    """old function for backwards compatibility; do not use this, use register_paste_params_button"""
+    for tabname, button in buttons.items():
+        source_text_component = send_generate_info if isinstance(send_generate_info, gr.components.Component) else None
+        source_tabname = send_generate_info if isinstance(send_generate_info, str) else None
+
+        register_paste_params_button(ParamBinding(paste_button=button, tabname=tabname, source_text_component=source_text_component, source_image_component=send_image, source_tabname=source_tabname))
+
+
+def register_paste_params_button(binding: ParamBinding):
+    registered_param_bindings.append(binding)
+
+
+def connect_paste_params_buttons():
+    binding: ParamBinding
+    for binding in registered_param_bindings:
+        destination_image_component = paste_fields[binding.tabname]["init_img"]
+        fields = paste_fields[binding.tabname]["fields"]
+        override_settings_component = binding.override_settings_component or paste_fields[binding.tabname]["override_settings_component"]
+
+        destination_width_component = next(iter([field for field, name in fields if name == "Size-1"] if fields else []), None)
+        destination_height_component = next(iter([field for field, name in fields if name == "Size-2"] if fields else []), None)
+
+        if binding.source_image_component and destination_image_component:
+            if isinstance(binding.source_image_component, gr.Gallery):
+                func = send_image_and_dimensions if destination_width_component else image_from_url_text
+                jsfunc = "extract_image_from_gallery"
+            else:
+                func = send_image_and_dimensions if destination_width_component else lambda x: x
+                jsfunc = None
+
+            binding.paste_button.click(
+                fn=func,
+                _js=jsfunc,
+                inputs=[binding.source_image_component],
+                outputs=[destination_image_component, destination_width_component, destination_height_component] if destination_width_component else [destination_image_component],
+                show_progress=False,
+            )
+
+        if binding.source_text_component is not None and fields is not None:
+            connect_paste(binding.paste_button, fields, binding.source_text_component, override_settings_component, binding.tabname)
+
+        if binding.source_tabname is not None and fields is not None:
+            paste_field_names = ['Prompt', 'Negative prompt', 'Steps', 'Face restoration'] + (["Seed"] if shared.opts.send_seed else []) + binding.paste_field_names
+            binding.paste_button.click(
+                fn=lambda *x: x,
+                inputs=[field for field, name in paste_fields[binding.source_tabname]["fields"] if name in paste_field_names],
+                outputs=[field for field, name in fields if name in paste_field_names],
+                show_progress=False,
+            )
+
+        binding.paste_button.click(
+            fn=None,
+            _js=f"switch_to_{binding.tabname}",
+            inputs=None,
+            outputs=None,
+            show_progress=False,
+        )
 
 
 def send_image_and_dimensions(x):
@@ -123,49 +174,6 @@ def send_image_and_dimensions(x):
     return img, w, h
 
 
-def run_bind():
-    for buttons, source_image_component, send_generate_info in bind_list:
-        for tab in buttons:
-            button = buttons[tab]
-            destination_image_component = paste_fields[tab]["init_img"]
-            fields = paste_fields[tab]["fields"]
-
-            destination_width_component = next(iter([field for field, name in fields if name == "Size-1"] if fields else []), None)
-            destination_height_component = next(iter([field for field, name in fields if name == "Size-2"] if fields else []), None)
-
-            if source_image_component and destination_image_component:
-                if isinstance(source_image_component, gr.Gallery):
-                    func = send_image_and_dimensions if destination_width_component else image_from_url_text
-                    jsfunc = "extract_image_from_gallery"
-                else:
-                    func = send_image_and_dimensions if destination_width_component else lambda x: x
-                    jsfunc = None
-
-                button.click(
-                    fn=func,
-                    _js=jsfunc,
-                    inputs=[source_image_component],
-                    outputs=[destination_image_component, destination_width_component, destination_height_component] if destination_width_component else [destination_image_component],
-                )
-
-            if send_generate_info and fields is not None:
-                if send_generate_info in paste_fields:
-                    paste_field_names = ['Prompt', 'Negative prompt', 'Steps', 'Face restoration'] + (["Seed"] if shared.opts.send_seed else [])
-                    button.click(
-                        fn=lambda *x: x,
-                        inputs=[field for field, name in paste_fields[send_generate_info]["fields"] if name in paste_field_names],
-                        outputs=[field for field, name in fields if name in paste_field_names],
-                    )
-                else:
-                    connect_paste(button, fields, send_generate_info)
-
-            button.click(
-                fn=None,
-                _js=f"switch_to_{tab}",
-                inputs=None,
-                outputs=None,
-            )
-
 
 def find_hypernetwork_key(hypernet_name, hypernet_hash=None):
     """Determines the config parameter name to use for the hypernet based on the parameters in the infotext.
@@ -243,31 +251,44 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model
     done_with_prompt = False
 
     *lines, lastline = x.strip().split("\n")
-    if not re_params.match(lastline):
+    if len(re_param.findall(lastline)) < 3:
         lines.append(lastline)
         lastline = ''
 
-    for i, line in enumerate(lines):
+    for line in lines:
         line = line.strip()
         if line.startswith("Negative prompt:"):
             done_with_prompt = True
             line = line[16:].strip()
-
         if done_with_prompt:
             negative_prompt += ("" if negative_prompt == "" else "\n") + line
         else:
             prompt += ("" if prompt == "" else "\n") + line
 
+    if shared.opts.infotext_styles != "Ignore":
+        found_styles, prompt, negative_prompt = shared.prompt_styles.extract_styles_from_prompt(prompt, negative_prompt)
+
+        if shared.opts.infotext_styles == "Apply":
+            res["Styles array"] = found_styles
+        elif shared.opts.infotext_styles == "Apply if any" and found_styles:
+            res["Styles array"] = found_styles
+
     res["Prompt"] = prompt
     res["Negative prompt"] = negative_prompt
 
     for k, v in re_param.findall(lastline):
-        m = re_imagesize.match(v)
-        if m is not None:
-            res[k+"-1"] = m.group(1)
-            res[k+"-2"] = m.group(2)
-        else:
-            res[k] = v
+        try:
+            if v[0] == '"' and v[-1] == '"':
+                v = unquote(v)
+
+            m = re_imagesize.match(v)
+            if m is not None:
+                res[f"{k}-1"] = m.group(1)
+                res[f"{k}-2"] = m.group(2)
+            else:
+                res[k] = v
+        except Exception:
+            print(f"Error parsing \"{k}: {v}\"")
 
     # Missing CLIP skip means it was set to 1 (the default)
     if "Clip skip" not in res:
@@ -281,15 +302,97 @@ Steps: 20, Sampler: Euler a, CFG scale: 7, Seed: 965400086, Size: 512x512, Model
         res["Hires resize-1"] = 0
         res["Hires resize-2"] = 0
 
+    if "Hires sampler" not in res:
+        res["Hires sampler"] = "Use same sampler"
+
+    if "Hires prompt" not in res:
+        res["Hires prompt"] = ""
+
+    if "Hires negative prompt" not in res:
+        res["Hires negative prompt"] = ""
+
     restore_old_hires_fix_params(res)
 
+    # Missing RNG means the default was set, which is GPU RNG
+    if "RNG" not in res:
+        res["RNG"] = "GPU"
+
+    if "Schedule type" not in res:
+        res["Schedule type"] = "Automatic"
+
+    if "Schedule max sigma" not in res:
+        res["Schedule max sigma"] = 0
+
+    if "Schedule min sigma" not in res:
+        res["Schedule min sigma"] = 0
+
+    if "Schedule rho" not in res:
+        res["Schedule rho"] = 0
+
     return res
 
 
-def connect_paste(button, paste_fields, input_comp, jsfunc=None):
+settings_map = {}
+
+
+
+infotext_to_setting_name_mapping = [
+    ('Clip skip', 'CLIP_stop_at_last_layers', ),
+    ('Conditional mask weight', 'inpainting_mask_weight'),
+    ('Model hash', 'sd_model_checkpoint'),
+    ('ENSD', 'eta_noise_seed_delta'),
+    ('Schedule type', 'k_sched_type'),
+    ('Schedule max sigma', 'sigma_max'),
+    ('Schedule min sigma', 'sigma_min'),
+    ('Schedule rho', 'rho'),
+    ('Noise multiplier', 'initial_noise_multiplier'),
+    ('Eta', 'eta_ancestral'),
+    ('Eta DDIM', 'eta_ddim'),
+    ('Discard penultimate sigma', 'always_discard_next_to_last_sigma'),
+    ('UniPC variant', 'uni_pc_variant'),
+    ('UniPC skip type', 'uni_pc_skip_type'),
+    ('UniPC order', 'uni_pc_order'),
+    ('UniPC lower order final', 'uni_pc_lower_order_final'),
+    ('Token merging ratio', 'token_merging_ratio'),
+    ('Token merging ratio hr', 'token_merging_ratio_hr'),
+    ('RNG', 'randn_source'),
+    ('NGMS', 's_min_uncond'),
+]
+
+
+def create_override_settings_dict(text_pairs):
+    """creates processing's override_settings parameters from gradio's multiselect
+
+    Example input:
+        ['Clip skip: 2', 'Model hash: e6e99610c4', 'ENSD: 31337']
+
+    Example output:
+        {'CLIP_stop_at_last_layers': 2, 'sd_model_checkpoint': 'e6e99610c4', 'eta_noise_seed_delta': 31337}
+    """
+
+    res = {}
+
+    params = {}
+    for pair in text_pairs:
+        k, v = pair.split(":", maxsplit=1)
+
+        params[k] = v.strip()
+
+    for param_name, setting_name in infotext_to_setting_name_mapping:
+        value = params.get(param_name, None)
+
+        if value is None:
+            continue
+
+        res[setting_name] = shared.opts.cast_value(setting_name, value)
+
+    return res
+
+
+def connect_paste(button, paste_fields, input_comp, override_settings_component, tabname):
     def paste_func(prompt):
         if not prompt and not shared.cmd_opts.hide_ui_dir_config:
-            filename = os.path.join(script_path, "params.txt")
+            filename = os.path.join(data_path, "params.txt")
             if os.path.exists(filename):
                 with open(filename, "r", encoding="utf8") as file:
                     prompt = file.read()
@@ -323,11 +426,42 @@ def connect_paste(button, paste_fields, input_comp, jsfunc=None):
 
         return res
 
+    if override_settings_component is not None:
+        def paste_settings(params):
+            vals = {}
+
+            for param_name, setting_name in infotext_to_setting_name_mapping:
+                v = params.get(param_name, None)
+                if v is None:
+                    continue
+
+                if setting_name == "sd_model_checkpoint" and shared.opts.disable_weights_auto_swap:
+                    continue
+
+                v = shared.opts.cast_value(setting_name, v)
+                current_value = getattr(shared.opts, setting_name, None)
+
+                if v == current_value:
+                    continue
+
+                vals[param_name] = v
+
+            vals_pairs = [f"{k}: {v}" for k, v in vals.items()]
+
+            return gr.Dropdown.update(value=vals_pairs, choices=vals_pairs, visible=bool(vals_pairs))
+
+        paste_fields = paste_fields + [(override_settings_component, paste_settings)]
+
     button.click(
         fn=paste_func,
-        _js=jsfunc,
         inputs=[input_comp],
         outputs=[x[0] for x in paste_fields],
+        show_progress=False,
+    )
+    button.click(
+        fn=None,
+        _js=f"recalculate_prompts_{tabname}",
+        inputs=[],
+        outputs=[],
+        show_progress=False,
     )
-
-

modules/gfpgan_model.py

@@ -1,17 +1,14 @@
 import os
-import sys
-import traceback
 
 import facexlib
 import gfpgan
 
 import modules.face_restoration
-from modules import shared, devices, modelloader
-from modules.paths import models_path
+from modules import paths, shared, devices, modelloader, errors
 
 model_dir = "GFPGAN"
 user_path = None
-model_path = os.path.join(models_path, model_dir)
+model_path = os.path.join(paths.models_path, model_dir)
 model_url = "https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth"
 have_gfpgan = False
 loaded_gfpgan_model = None
@@ -79,7 +76,7 @@ def setup_model(dirname):
 
     try:
         from gfpgan import GFPGANer
-        from facexlib import detection, parsing
+        from facexlib import detection, parsing  # noqa: F401
         global user_path
         global have_gfpgan
         global gfpgan_constructor
@@ -113,5 +110,4 @@ def setup_model(dirname):
 
         shared.face_restorers.append(FaceRestorerGFPGAN())
     except Exception:
-        print("Error setting up GFPGAN:", file=sys.stderr)
-        print(traceback.format_exc(), file=sys.stderr)
+        errors.report("Error setting up GFPGAN", exc_info=True)

modules/gitpython_hack.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import io
+import subprocess
+
+import git
+
+
+class Git(git.Git):
+    """
+    Git subclassed to never use persistent processes.
+    """
+
+    def _get_persistent_cmd(self, attr_name, cmd_name, *args, **kwargs):
+        raise NotImplementedError(f"Refusing to use persistent process: {attr_name} ({cmd_name} {args} {kwargs})")
+
+    def get_object_header(self, ref: str | bytes) -> tuple[str, str, int]:
+        ret = subprocess.check_output(
+            [self.GIT_PYTHON_GIT_EXECUTABLE, "cat-file", "--batch-check"],
+            input=self._prepare_ref(ref),
+            cwd=self._working_dir,
+            timeout=2,
+        )
+        return self._parse_object_header(ret)
+
+    def stream_object_data(self, ref: str) -> tuple[str, str, int, "Git.CatFileContentStream"]:
+        # Not really streaming, per se; this buffers the entire object in memory.
+        # Shouldn't be a problem for our use case, since we're only using this for
+        # object headers (commit objects).
+        ret = subprocess.check_output(
+            [self.GIT_PYTHON_GIT_EXECUTABLE, "cat-file", "--batch"],
+            input=self._prepare_ref(ref),
+            cwd=self._working_dir,
+            timeout=30,
+        )
+        bio = io.BytesIO(ret)
+        hexsha, typename, size = self._parse_object_header(bio.readline())
+        return (hexsha, typename, size, self.CatFileContentStream(size, bio))
+
+
+class Repo(git.Repo):
+    GitCommandWrapperType = Git

modules/hashes.py

@@ -4,13 +4,16 @@ import os.path
 
 import filelock
 
+from modules import shared
+from modules.paths import data_path
 
-cache_filename = "cache.json"
+
+cache_filename = os.path.join(data_path, "cache.json")
 cache_data = None
 
 
 def dump_cache():
-    with filelock.FileLock(cache_filename+".lock"):
+    with filelock.FileLock(f"{cache_filename}.lock"):
         with open(cache_filename, "w", encoding="utf8") as file:
             json.dump(cache_data, file, indent=4)
 
@@ -19,7 +22,7 @@ def cache(subsection):
     global cache_data
 
     if cache_data is None:
-        with filelock.FileLock(cache_filename+".lock"):
+        with filelock.FileLock(f"{cache_filename}.lock"):
             if not os.path.isfile(cache_filename):
                 cache_data = {}
             else:
@@ -43,8 +46,8 @@ def calculate_sha256(filename):
     return hash_sha256.hexdigest()
 
 
-def sha256_from_cache(filename, title):
-    hashes = cache("hashes")
+def sha256_from_cache(filename, title, use_addnet_hash=False):
+    hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes")
     ondisk_mtime = os.path.getmtime(filename)
 
     if title not in hashes:
@@ -59,15 +62,22 @@ def sha256_from_cache(filename, title):
     return cached_sha256
 
 
-def sha256(filename, title):
-    hashes = cache("hashes")
+def sha256(filename, title, use_addnet_hash=False):
+    hashes = cache("hashes-addnet") if use_addnet_hash else cache("hashes")
 
-    sha256_value = sha256_from_cache(filename, title)
+    sha256_value = sha256_from_cache(filename, title, use_addnet_hash)
     if sha256_value is not None:
         return sha256_value
 
+    if shared.cmd_opts.no_hashing:
+        return None
+
     print(f"Calculating sha256 for {filename}: ", end='')
-    sha256_value = calculate_sha256(filename)
+    if use_addnet_hash:
+        with open(filename, "rb") as file:
+            sha256_value = addnet_hash_safetensors(file)
+    else:
+        sha256_value = calculate_sha256(filename)
     print(f"{sha256_value}")
 
     hashes[title] = {
@@ -80,6 +90,19 @@ def sha256(filename, title):
     return sha256_value
 
 
+def addnet_hash_safetensors(b):
+    """kohya-ss hash for safetensors from https://github.com/kohya-ss/sd-scripts/blob/main/library/train_util.py"""
+    hash_sha256 = hashlib.sha256()
+    blksize = 1024 * 1024
 
+    b.seek(0)
+    header = b.read(8)
+    n = int.from_bytes(header, "little")
 
+    offset = n + 8
+    b.seek(offset)
+    for chunk in iter(lambda: b.read(blksize), b""):
+        hash_sha256.update(chunk)
+
+    return hash_sha256.hexdigest()
 

modules/hypernetworks/hypernetwork.py

@@ -1,10 +1,7 @@
-import csv
 import datetime
 import glob
 import html
 import os
-import sys
-import traceback
 import inspect
 
 import modules.textual_inversion.dataset
@@ -12,13 +9,13 @@ import torch
 import tqdm
 from einops import rearrange, repeat
 from ldm.util import default
-from modules import devices, processing, sd_models, shared, sd_samplers, hashes, sd_hijack_checkpoint
+from modules import devices, processing, sd_models, shared, sd_samplers, hashes, sd_hijack_checkpoint, errors
 from modules.textual_inversion import textual_inversion, logging
 from modules.textual_inversion.learn_schedule import LearnRateScheduler
 from torch import einsum
 from torch.nn.init import normal_, xavier_normal_, xavier_uniform_, kaiming_normal_, kaiming_uniform_, zeros_
 
-from collections import defaultdict, deque
+from collections import deque
 from statistics import stdev, mean
 
 
@@ -178,34 +175,34 @@ class Hypernetwork:
 
     def weights(self):
         res = []
-        for k, layers in self.layers.items():
+        for layers in self.layers.values():
             for layer in layers:
                 res += layer.parameters()
         return res
 
     def train(self, mode=True):
-        for k, layers in self.layers.items():
+        for layers in self.layers.values():
             for layer in layers:
                 layer.train(mode=mode)
                 for param in layer.parameters():
                     param.requires_grad = mode
 
     def to(self, device):
-        for k, layers in self.layers.items():
+        for layers in self.layers.values():
             for layer in layers:
                 layer.to(device)
 
         return self
 
     def set_multiplier(self, multiplier):
-        for k, layers in self.layers.items():
+        for layers in self.layers.values():
             for layer in layers:
                 layer.multiplier = multiplier
 
         return self
 
     def eval(self):
-        for k, layers in self.layers.items():
+        for layers in self.layers.values():
             for layer in layers:
                 layer.eval()
                 for param in layer.parameters():
@@ -307,12 +304,12 @@ class Hypernetwork:
     def shorthash(self):
         sha256 = hashes.sha256(self.filename, f'hypernet/{self.name}')
 
-        return sha256[0:10]
+        return sha256[0:10] if sha256 else None
 
 
 def list_hypernetworks(path):
     res = {}
-    for filename in sorted(glob.iglob(os.path.join(path, '**/*.pt'), recursive=True)):
+    for filename in sorted(glob.iglob(os.path.join(path, '**/*.pt'), recursive=True), key=str.lower):
         name = os.path.splitext(os.path.basename(filename))[0]
         # Prevent a hypothetical "None.pt" from being listed.
         if name != "None":
@@ -326,17 +323,14 @@ def load_hypernetwork(name):
     if path is None:
         return None
 
-    hypernetwork = Hypernetwork()
-
     try:
+        hypernetwork = Hypernetwork()
         hypernetwork.load(path)
+        return hypernetwork
     except Exception:
-        print(f"Error loading hypernetwork {path}", file=sys.stderr)
-        print(traceback.format_exc(), file=sys.stderr)
+        errors.report(f"Error loading hypernetwork {path}", exc_info=True)
         return None
 
-    return hypernetwork
-
 
 def load_hypernetworks(names, multipliers=None):
     already_loaded = {}
@@ -380,8 +374,8 @@ def apply_single_hypernetwork(hypernetwork, context_k, context_v, layer=None):
         layer.hyper_k = hypernetwork_layers[0]
         layer.hyper_v = hypernetwork_layers[1]
 
-    context_k = hypernetwork_layers[0](context_k)
-    context_v = hypernetwork_layers[1](context_v)
+    context_k = devices.cond_cast_unet(hypernetwork_layers[0](devices.cond_cast_float(context_k)))
+    context_v = devices.cond_cast_unet(hypernetwork_layers[1](devices.cond_cast_float(context_v)))
     return context_k, context_v
 
 
@@ -404,7 +398,7 @@ def attention_CrossAttention_forward(self, x, context=None, mask=None):
     k = self.to_k(context_k)
     v = self.to_v(context_v)
 
-    q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v))
+    q, k, v = (rearrange(t, 'b n (h d) -> (b h) n d', h=h) for t in (q, k, v))
 
     sim = einsum('b i d, b j d -> b i j', q, k) * self.scale
 
@@ -496,7 +490,7 @@ def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None,
     shared.reload_hypernetworks()
 
 
-def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, create_image_every, save_hypernetwork_every, template_filename, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
+def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradient_step, data_root, log_directory, training_width, training_height, varsize, steps, clip_grad_mode, clip_grad_value, shuffle_tags, tag_drop_out, latent_sampling_method, use_weight, create_image_every, save_hypernetwork_every, template_filename, preview_from_txt2img, preview_prompt, preview_negative_prompt, preview_steps, preview_sampler_index, preview_cfg_scale, preview_seed, preview_width, preview_height):
     # images allows training previews to have infotext. Importing it at the top causes a circular import problem.
     from modules import images
 
@@ -541,7 +535,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
         return hypernetwork, filename
 
     scheduler = LearnRateScheduler(learn_rate, steps, initial_step)
-    
+
     clip_grad = torch.nn.utils.clip_grad_value_ if clip_grad_mode == "value" else torch.nn.utils.clip_grad_norm_ if clip_grad_mode == "norm" else None
     if clip_grad:
         clip_grad_sched = LearnRateScheduler(clip_grad_value, steps, initial_step, verbose=False)
@@ -554,7 +548,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
 
     pin_memory = shared.opts.pin_memory
 
-    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize)
+    ds = modules.textual_inversion.dataset.PersonalizedBase(data_root=data_root, width=training_width, height=training_height, repeats=shared.opts.training_image_repeats_per_epoch, placeholder_token=hypernetwork_name, model=shared.sd_model, cond_model=shared.sd_model.cond_stage_model, device=devices.device, template_file=template_file, include_cond=True, batch_size=batch_size, gradient_step=gradient_step, shuffle_tags=shuffle_tags, tag_drop_out=tag_drop_out, latent_sampling_method=latent_sampling_method, varsize=varsize, use_weight=use_weight)
 
     if shared.opts.save_training_settings_to_txt:
         saved_params = dict(
@@ -594,7 +588,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
             print(e)
 
     scaler = torch.cuda.amp.GradScaler()
-    
+
     batch_size = ds.batch_size
     gradient_step = ds.gradient_step
     # n steps = batch_size * gradient_step * n image processed
@@ -620,7 +614,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
     try:
         sd_hijack_checkpoint.add()
 
-        for i in range((steps-initial_step) * gradient_step):
+        for _ in range((steps-initial_step) * gradient_step):
             if scheduler.finished:
                 break
             if shared.state.interrupted:
@@ -637,29 +631,35 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
 
                 if clip_grad:
                     clip_grad_sched.step(hypernetwork.step)
-                
+
                 with devices.autocast():
                     x = batch.latent_sample.to(devices.device, non_blocking=pin_memory)
+                    if use_weight:
+                        w = batch.weight.to(devices.device, non_blocking=pin_memory)
                     if tag_drop_out != 0 or shuffle_tags:
                         shared.sd_model.cond_stage_model.to(devices.device)
                         c = shared.sd_model.cond_stage_model(batch.cond_text).to(devices.device, non_blocking=pin_memory)
                         shared.sd_model.cond_stage_model.to(devices.cpu)
                     else:
                         c = stack_conds(batch.cond).to(devices.device, non_blocking=pin_memory)
-                    loss = shared.sd_model(x, c)[0] / gradient_step
+                    if use_weight:
+                        loss = shared.sd_model.weighted_forward(x, c, w)[0] / gradient_step
+                        del w
+                    else:
+                        loss = shared.sd_model.forward(x, c)[0] / gradient_step
                     del x
                     del c
 
                     _loss_step += loss.item()
                 scaler.scale(loss).backward()
-                
+
                 # go back until we reach gradient accumulation steps
                 if (j + 1) % gradient_step != 0:
                     continue
                 loss_logging.append(_loss_step)
                 if clip_grad:
                     clip_grad(weights, clip_grad_sched.learn_rate)
-                
+
                 scaler.step(optimizer)
                 scaler.update()
                 hypernetwork.step += 1
@@ -669,7 +669,7 @@ def train_hypernetwork(id_task, hypernetwork_name, learn_rate, batch_size, gradi
                 _loss_step = 0
 
                 steps_done = hypernetwork.step + 1
-                
+
                 epoch_num = hypernetwork.step // steps_per_epoch
                 epoch_step = hypernetwork.step % steps_per_epoch
 
@@ -765,7 +765,7 @@ Last saved image: {html.escape(last_saved_image)}<br/>
 </p>
 """
     except Exception:
-        print(traceback.format_exc(), file=sys.stderr)
+        errors.report("Exception in training hypernetwork", exc_info=True)
     finally:
         pbar.leave = False
         pbar.close()

modules/hypernetworks/ui.py

@@ -1,19 +1,17 @@
 import html
-import os
-import re
 
 import gradio as gr
 import modules.hypernetworks.hypernetwork
 from modules import devices, sd_hijack, shared
 
 not_available = ["hardswish", "multiheadattention"]
-keys = list(x for x in modules.hypernetworks.hypernetwork.HypernetworkModule.activation_dict.keys() if x not in not_available)
+keys = [x for x in modules.hypernetworks.hypernetwork.HypernetworkModule.activation_dict if x not in not_available]
 
 
 def create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure=None, activation_func=None, weight_init=None, add_layer_norm=False, use_dropout=False, dropout_structure=None):
     filename = modules.hypernetworks.hypernetwork.create_hypernetwork(name, enable_sizes, overwrite_old, layer_structure, activation_func, weight_init, add_layer_norm, use_dropout, dropout_structure)
 
-    return gr.Dropdown.update(choices=sorted([x for x in shared.hypernetworks.keys()])), f"Created: {filename}", ""
+    return gr.Dropdown.update(choices=sorted(shared.hypernetworks)), f"Created: {filename}", ""
 
 
 def train_hypernetwork(*args):

modules/images.py

@@ -1,6 +1,4 @@
 import datetime
-import sys
-import traceback
 
 import pytz
 import io
@@ -13,16 +11,26 @@ import numpy as np
 import piexif
 import piexif.helper
 from PIL import Image, ImageFont, ImageDraw, PngImagePlugin
-from fonts.ttf import Roboto
 import string
 import json
+import hashlib
 
-from modules import sd_samplers, shared, script_callbacks
-from modules.shared import opts, cmd_opts
+from modules import sd_samplers, shared, script_callbacks, errors
+from modules.paths_internal import roboto_ttf_file
+from modules.shared import opts
+
+import modules.sd_vae as sd_vae
 
 LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS)
 
 
+def get_font(fontsize: int):
+    try:
+        return ImageFont.truetype(opts.font or roboto_ttf_file, fontsize)
+    except Exception:
+        return ImageFont.truetype(roboto_ttf_file, fontsize)
+
+
 def image_grid(imgs, batch_size=1, rows=None):
     if rows is None:
         if opts.n_rows > 0:
@@ -36,6 +44,8 @@ def image_grid(imgs, batch_size=1, rows=None):
         else:
             rows = math.sqrt(len(imgs))
             rows = round(rows)
+    if rows > len(imgs):
+        rows = len(imgs)
 
     cols = math.ceil(len(imgs) / rows)
 
@@ -128,7 +138,7 @@ class GridAnnotation:
         self.size = None
 
 
-def draw_grid_annotations(im, width, height, hor_texts, ver_texts):
+def draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin=0):
     def wrap(drawing, text, font, line_length):
         lines = ['']
         for word in text.split():
@@ -139,14 +149,8 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts):
                 lines.append(word)
         return lines
 
-    def get_font(fontsize):
-        try:
-            return ImageFont.truetype(opts.font or Roboto, fontsize)
-        except Exception:
-            return ImageFont.truetype(Roboto, fontsize)
-
     def draw_texts(drawing, draw_x, draw_y, lines, initial_fnt, initial_fontsize):
-        for i, line in enumerate(lines):
+        for line in lines:
             fnt = initial_fnt
             fontsize = initial_fontsize
             while drawing.multiline_textsize(line.text, font=fnt)[0] > line.allowed_width and fontsize > 0:
@@ -192,32 +196,35 @@ def draw_grid_annotations(im, width, height, hor_texts, ver_texts):
             line.allowed_width = allowed_width
 
     hor_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing for lines in hor_texts]
-    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in
-                        ver_texts]
+    ver_text_heights = [sum([line.size[1] + line_spacing for line in lines]) - line_spacing * len(lines) for lines in ver_texts]
 
-    pad_top = max(hor_text_heights) + line_spacing * 2
+    pad_top = 0 if sum(hor_text_heights) == 0 else max(hor_text_heights) + line_spacing * 2
 
-    result = Image.new("RGB", (im.width + pad_left, im.height + pad_top), "white")
-    result.paste(im, (pad_left, pad_top))
+    result = Image.new("RGB", (im.width + pad_left + margin * (cols-1), im.height + pad_top + margin * (rows-1)), "white")
+
+    for row in range(rows):
+        for col in range(cols):
+            cell = im.crop((width * col, height * row, width * (col+1), height * (row+1)))
+            result.paste(cell, (pad_left + (width + margin) * col, pad_top + (height + margin) * row))
 
     d = ImageDraw.Draw(result)
 
     for col in range(cols):
-        x = pad_left + width * col + width / 2
+        x = pad_left + (width + margin) * col + width / 2
         y = pad_top / 2 - hor_text_heights[col] / 2
 
         draw_texts(d, x, y, hor_texts[col], fnt, fontsize)
 
     for row in range(rows):
         x = pad_left / 2
-        y = pad_top + height * row + height / 2 - ver_text_heights[row] / 2
+        y = pad_top + (height + margin) * row + height / 2 - ver_text_heights[row] / 2
 
         draw_texts(d, x, y, ver_texts[row], fnt, fontsize)
 
     return result
 
 
-def draw_prompt_matrix(im, width, height, all_prompts):
+def draw_prompt_matrix(im, width, height, all_prompts, margin=0):
     prompts = all_prompts[1:]
     boundary = math.ceil(len(prompts) / 2)
 
@@ -227,7 +234,7 @@ def draw_prompt_matrix(im, width, height, all_prompts):
     hor_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_horiz)] for pos in range(1 << len(prompts_horiz))]
     ver_texts = [[GridAnnotation(x, is_active=pos & (1 << i) != 0) for i, x in enumerate(prompts_vert)] for pos in range(1 << len(prompts_vert))]
 
-    return draw_grid_annotations(im, width, height, hor_texts, ver_texts)
+    return draw_grid_annotations(im, width, height, hor_texts, ver_texts, margin)
 
 
 def resize_image(resize_mode, im, width, height, upscaler_name=None):
@@ -255,9 +262,12 @@ def resize_image(resize_mode, im, width, height, upscaler_name=None):
 
         if scale > 1.0:
             upscalers = [x for x in shared.sd_upscalers if x.name == upscaler_name]
-            assert len(upscalers) > 0, f"could not find upscaler named {upscaler_name}"
+            if len(upscalers) == 0:
+                upscaler = shared.sd_upscalers[0]
+                print(f"could not find upscaler named {upscaler_name or '<empty string>'}, using {upscaler.name} as a fallback")
+            else:
+                upscaler = upscalers[0]
 
-            upscaler = upscalers[0]
             im = upscaler.scaler.upscale(im, scale, upscaler.data_path)
 
         if im.width != w or im.height != h:
@@ -309,6 +319,7 @@ re_nonletters = re.compile(r'[\s' + string.punctuation + ']+')
 re_pattern = re.compile(r"(.*?)(?:\[([^\[\]]+)\]|$)")
 re_pattern_arg = re.compile(r"(.*)<([^>]*)>$")
 max_filename_part_length = 128
+NOTHING_AND_SKIP_PREVIOUS_TEXT = object()
 
 
 def sanitize_filename_part(text, replace_spaces=True):
@@ -325,8 +336,20 @@ def sanitize_filename_part(text, replace_spaces=True):
 
 
 class FilenameGenerator:
+    def get_vae_filename(self): #get the name of the VAE file.
+        if sd_vae.loaded_vae_file is None:
+            return "NoneType"
+        file_name = os.path.basename(sd_vae.loaded_vae_file)
+        split_file_name = file_name.split('.')
+        if len(split_file_name) > 1 and split_file_name[0] == '':
+            return split_file_name[1] # if the first character of the filename is "." then [1] is obtained.
+        else:
+            return split_file_name[0]
+
     replacements = {
         'seed': lambda self: self.seed if self.seed is not None else '',
+        'seed_first': lambda self: self.seed if self.p.batch_size == 1 else self.p.all_seeds[0],
+        'seed_last': lambda self: NOTHING_AND_SKIP_PREVIOUS_TEXT if self.p.batch_size == 1 else self.p.all_seeds[-1],
         'steps': lambda self:  self.p and self.p.steps,
         'cfg': lambda self: self.p and self.p.cfg_scale,
         'width': lambda self: self.image.width,
@@ -338,18 +361,44 @@ class FilenameGenerator:
         'date': lambda self: datetime.datetime.now().strftime('%Y-%m-%d'),
         'datetime': lambda self, *args: self.datetime(*args),  # accepts formats: [datetime], [datetime<Format>], [datetime<Format><Time Zone>]
         'job_timestamp': lambda self: getattr(self.p, "job_timestamp", shared.state.job_timestamp),
+        'prompt_hash': lambda self: hashlib.sha256(self.prompt.encode()).hexdigest()[0:8],
         'prompt': lambda self: sanitize_filename_part(self.prompt),
         'prompt_no_styles': lambda self: self.prompt_no_style(),
         'prompt_spaces': lambda self: sanitize_filename_part(self.prompt, replace_spaces=False),
         'prompt_words': lambda self: self.prompt_words(),
+        'batch_number': lambda self: NOTHING_AND_SKIP_PREVIOUS_TEXT if self.p.batch_size == 1 or self.zip else self.p.batch_index + 1,
+        'batch_size': lambda self: self.p.batch_size,
+        'generation_number': lambda self: NOTHING_AND_SKIP_PREVIOUS_TEXT if (self.p.n_iter == 1 and self.p.batch_size == 1) or self.zip else self.p.iteration * self.p.batch_size + self.p.batch_index + 1,
+        'hasprompt': lambda self, *args: self.hasprompt(*args),  # accepts formats:[hasprompt<prompt1|default><prompt2>..]
+        'clip_skip': lambda self: opts.data["CLIP_stop_at_last_layers"],
+        'denoising': lambda self: self.p.denoising_strength if self.p and self.p.denoising_strength else NOTHING_AND_SKIP_PREVIOUS_TEXT,
+        'vae_filename': lambda self: self.get_vae_filename(),
+
     }
     default_time_format = '%Y%m%d%H%M%S'
 
-    def __init__(self, p, seed, prompt, image):
+    def __init__(self, p, seed, prompt, image, zip=False):
         self.p = p
         self.seed = seed
         self.prompt = prompt
         self.image = image
+        self.zip = zip
+
+    def hasprompt(self, *args):
+        lower = self.prompt.lower()
+        if self.p is None or self.prompt is None:
+            return None
+        outres = ""
+        for arg in args:
+            if arg != "":
+                division = arg.split("|")
+                expected = division[0].lower()
+                default = division[1] if len(division) > 1 else ""
+                if lower.find(expected) >= 0:
+                    outres = f'{outres}{expected}'
+                else:
+                    outres = outres if default == "" else f'{outres}{default}'
+        return sanitize_filename_part(outres)
 
     def prompt_no_style(self):
         if self.p is None or self.prompt is None:
@@ -357,7 +406,7 @@ class FilenameGenerator:
 
         prompt_no_style = self.prompt
         for style in shared.prompt_styles.get_style_prompts(self.p.styles):
-            if len(style) > 0:
+            if style:
                 for part in style.split("{prompt}"):
                     prompt_no_style = prompt_no_style.replace(part, "").replace(", ,", ",").strip().strip(',')
 
@@ -366,7 +415,7 @@ class FilenameGenerator:
         return sanitize_filename_part(prompt_no_style, replace_spaces=False)
 
     def prompt_words(self):
-        words = [x for x in re_nonletters.split(self.prompt or "") if len(x) > 0]
+        words = [x for x in re_nonletters.split(self.prompt or "") if x]
         if len(words) == 0:
             words = ["empty"]
         return sanitize_filename_part(" ".join(words[0:opts.directories_max_prompt_words]), replace_spaces=False)
@@ -374,16 +423,16 @@ class FilenameGenerator:
     def datetime(self, *args):
         time_datetime = datetime.datetime.now()
 
-        time_format = args[0] if len(args) > 0 and args[0] != "" else self.default_time_format
+        time_format = args[0] if (args and args[0] != "") else self.default_time_format
         try:
             time_zone = pytz.timezone(args[1]) if len(args) > 1 else None
-        except pytz.exceptions.UnknownTimeZoneError as _:
+        except pytz.exceptions.UnknownTimeZoneError:
             time_zone = None
 
         time_zone_time = time_datetime.astimezone(time_zone)
         try:
             formatted_time = time_zone_time.strftime(time_format)
-        except (ValueError, TypeError) as _:
+        except (ValueError, TypeError):
             formatted_time = time_zone_time.strftime(self.default_time_format)
 
         return sanitize_filename_part(formatted_time, replace_spaces=False)
@@ -393,9 +442,9 @@ class FilenameGenerator:
 
         for m in re_pattern.finditer(x):
             text, pattern = m.groups()
-            res += text
 
             if pattern is None:
+                res += text
                 continue
 
             pattern_args = []
@@ -413,14 +462,15 @@ class FilenameGenerator:
                     replacement = fun(self, *pattern_args)
                 except Exception:
                     replacement = None
-                    print(f"Error adding [{pattern}] to filename", file=sys.stderr)
-                    print(traceback.format_exc(), file=sys.stderr)
+                    errors.report(f"Error adding [{pattern}] to filename", exc_info=True)
 
-                if replacement is not None:
-                    res += str(replacement)
+                if replacement == NOTHING_AND_SKIP_PREVIOUS_TEXT:
+                    continue
+                elif replacement is not None:
+                    res += text + str(replacement)
                     continue
 
-            res += f'[{pattern}]'
+            res += f'{text}[{pattern}]'
 
         return res
 
@@ -433,20 +483,56 @@ def get_next_sequence_number(path, basename):
     """
     result = -1
     if basename != '':
-        basename = basename + "-"
+        basename = f"{basename}-"
 
     prefix_length = len(basename)
     for p in os.listdir(path):
         if p.startswith(basename):
-            l = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
+            parts = os.path.splitext(p[prefix_length:])[0].split('-')  # splits the filename (removing the basename first if one is defined, so the sequence number is always the first element)
             try:
-                result = max(int(l[0]), result)
+                result = max(int(parts[0]), result)
             except ValueError:
                 pass
 
     return result + 1
 
 
+def save_image_with_geninfo(image, geninfo, filename, extension=None, existing_pnginfo=None):
+    if extension is None:
+        extension = os.path.splitext(filename)[1]
+
+    image_format = Image.registered_extensions()[extension]
+
+    if extension.lower() == '.png':
+        if opts.enable_pnginfo:
+            pnginfo_data = PngImagePlugin.PngInfo()
+            for k, v in (existing_pnginfo or {}).items():
+                pnginfo_data.add_text(k, str(v))
+        else:
+            pnginfo_data = None
+
+        image.save(filename, format=image_format, quality=opts.jpeg_quality, pnginfo=pnginfo_data)
+
+    elif extension.lower() in (".jpg", ".jpeg", ".webp"):
+        if image.mode == 'RGBA':
+            image = image.convert("RGB")
+        elif image.mode == 'I;16':
+            image = image.point(lambda p: p * 0.0038910505836576).convert("RGB" if extension.lower() == ".webp" else "L")
+
+        image.save(filename, format=image_format, quality=opts.jpeg_quality, lossless=opts.webp_lossless)
+
+        if opts.enable_pnginfo and geninfo is not None:
+            exif_bytes = piexif.dump({
+                "Exif": {
+                    piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(geninfo or "", encoding="unicode")
+                },
+            })
+
+            piexif.insert(exif_bytes, filename)
+    else:
+        image.save(filename, format=image_format, quality=opts.jpeg_quality)
+
+
 def save_image(image, path, basename, seed=None, prompt=None, extension='png', info=None, short_filename=False, no_prompt=False, grid=False, pnginfo_section_name='parameters', p=None, existing_info=None, forced_filename=None, suffix="", save_to_dirs=None):
     """Save an image.
 
@@ -502,7 +588,7 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='png', i
         add_number = opts.save_images_add_number or file_decoration == ''
 
         if file_decoration != "" and add_number:
-            file_decoration = "-" + file_decoration
+            file_decoration = f"-{file_decoration}"
 
         file_decoration = namegen.apply(file_decoration) + suffix
 
@@ -531,59 +617,43 @@ def save_image(image, path, basename, seed=None, prompt=None, extension='png', i
     info = params.pnginfo.get(pnginfo_section_name, None)
 
     def _atomically_save_image(image_to_save, filename_without_extension, extension):
-        # save image with .tmp extension to avoid race condition when another process detects new image in the directory
-        temp_file_path = filename_without_extension + ".tmp"
-        image_format = Image.registered_extensions()[extension]
+        """
+        save image with .tmp extension to avoid race condition when another process detects new image in the directory
+        """
+        temp_file_path = f"{filename_without_extension}.tmp"
 
-        if extension.lower() == '.png':
-            pnginfo_data = PngImagePlugin.PngInfo()
-            if opts.enable_pnginfo:
-                for k, v in params.pnginfo.items():
-                    pnginfo_data.add_text(k, str(v))
+        save_image_with_geninfo(image_to_save, info, temp_file_path, extension, params.pnginfo)
 
-            image_to_save.save(temp_file_path, format=image_format, quality=opts.jpeg_quality, pnginfo=pnginfo_data)
-
-        elif extension.lower() in (".jpg", ".jpeg", ".webp"):
-            if image_to_save.mode == 'RGBA':
-                image_to_save = image_to_save.convert("RGB")
-
-            image_to_save.save(temp_file_path, format=image_format, quality=opts.jpeg_quality)
-
-            if opts.enable_pnginfo and info is not None:
-                exif_bytes = piexif.dump({
-                    "Exif": {
-                        piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(info or "", encoding="unicode")
-                    },
-                })
-
-                piexif.insert(exif_bytes, temp_file_path)
-        else:
-            image_to_save.save(temp_file_path, format=image_format, quality=opts.jpeg_quality)
-
-        # atomically rename the file with correct extension
         os.replace(temp_file_path, filename_without_extension + extension)
 
     fullfn_without_extension, extension = os.path.splitext(params.filename)
+    if hasattr(os, 'statvfs'):
+        max_name_len = os.statvfs(path).f_namemax
+        fullfn_without_extension = fullfn_without_extension[:max_name_len - max(4, len(extension))]
+        params.filename = fullfn_without_extension + extension
+        fullfn = params.filename
     _atomically_save_image(image, fullfn_without_extension, extension)
 
     image.already_saved_as = fullfn
 
-    target_side_length = 4000
-    oversize = image.width > target_side_length or image.height > target_side_length
-    if opts.export_for_4chan and (oversize or os.stat(fullfn).st_size > 4 * 1024 * 1024):
+    oversize = image.width > opts.target_side_length or image.height > opts.target_side_length
+    if opts.export_for_4chan and (oversize or os.stat(fullfn).st_size > opts.img_downscale_threshold * 1024 * 1024):
         ratio = image.width / image.height
 
         if oversize and ratio > 1:
-            image = image.resize((target_side_length, image.height * target_side_length // image.width), LANCZOS)
+            image = image.resize((round(opts.target_side_length), round(image.height * opts.target_side_length / image.width)), LANCZOS)
         elif oversize:
-            image = image.resize((image.width * target_side_length // image.height, target_side_length), LANCZOS)
+            image = image.resize((round(image.width * opts.target_side_length / image.height), round(opts.target_side_length)), LANCZOS)
 
-        _atomically_save_image(image, fullfn_without_extension, ".jpg")
+        try:
+            _atomically_save_image(image, fullfn_without_extension, ".jpg")
+        except Exception as e:
+            errors.display(e, "saving image as downscaled JPG")
 
     if opts.save_txt and info is not None:
         txt_fullfn = f"{fullfn_without_extension}.txt"
         with open(txt_fullfn, "w", encoding="utf8") as file:
-            file.write(info + "\n")
+            file.write(f"{info}\n")
     else:
         txt_fullfn = None
 
@@ -609,9 +679,10 @@ def read_info_from_image(image):
             items['exif comment'] = exif_comment
             geninfo = exif_comment
 
-        for field in ['jfif', 'jfif_version', 'jfif_unit', 'jfif_density', 'dpi', 'exif',
-                      'loop', 'background', 'timestamp', 'duration']:
-            items.pop(field, None)
+    for field in ['jfif', 'jfif_version', 'jfif_unit', 'jfif_density', 'dpi', 'exif',
+                    'loop', 'background', 'timestamp', 'duration', 'progressive', 'progression',
+                    'icc_profile', 'chromaticity']:
+        items.pop(field, None)
 
     if items.get("Software", None) == "NovelAI":
         try:
@@ -622,13 +693,14 @@ def read_info_from_image(image):
 Negative prompt: {json_info["uc"]}
 Steps: {json_info["steps"]}, Sampler: {sampler}, CFG scale: {json_info["scale"]}, Seed: {json_info["seed"]}, Size: {image.width}x{image.height}, Clip skip: 2, ENSD: 31337"""
         except Exception:
-            print("Error parsing NovelAI image generation parameters:", file=sys.stderr)
-            print(traceback.format_exc(), file=sys.stderr)
+            errors.report("Error parsing NovelAI image generation parameters", exc_info=True)
 
     return geninfo, items
 
 
 def image_data(data):
+    import gradio as gr
+
     try:
         image = Image.open(io.BytesIO(data))
         textinfo, _ = read_info_from_image(image)
@@ -644,7 +716,7 @@ def image_data(data):
     except Exception:
         pass
 
-    return '', None
+    return gr.update(), None
 
 
 def flatten(img, bgcolor):

modules/img2img.py

@@ -1,26 +1,32 @@
-import math
 import os
-import sys
-import traceback
+from pathlib import Path
 
 import numpy as np
-from PIL import Image, ImageOps, ImageFilter, ImageEnhance, ImageChops
+from PIL import Image, ImageOps, ImageFilter, ImageEnhance, ImageChops, UnidentifiedImageError
 
-from modules import devices, sd_samplers
+from modules import sd_samplers
+from modules.generation_parameters_copypaste import create_override_settings_dict
 from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images
 from modules.shared import opts, state
 import modules.shared as shared
 import modules.processing as processing
 from modules.ui import plaintext_to_html
-import modules.images as images
 import modules.scripts
 
 
-def process_batch(p, input_dir, output_dir, args):
+def process_batch(p, input_dir, output_dir, inpaint_mask_dir, args, to_scale=False, scale_by=1.0):
     processing.fix_seed(p)
 
     images = shared.listfiles(input_dir)
 
+    is_inpaint_batch = False
+    if inpaint_mask_dir:
+        inpaint_masks = shared.listfiles(inpaint_mask_dir)
+        is_inpaint_batch = bool(inpaint_masks)
+
+        if is_inpaint_batch:
+            print(f"\nInpaint batch is enabled. {len(inpaint_masks)} masks found.")
+
     print(f"Will process {len(images)} images, creating {p.n_iter * p.batch_size} new images for each.")
 
     save_normally = output_dir == ''
@@ -38,17 +44,47 @@ def process_batch(p, input_dir, output_dir, args):
         if state.interrupted:
             break
 
-        img = Image.open(image)
+        try:
+            img = Image.open(image)
+        except UnidentifiedImageError as e:
+            print(e)
+            continue
         # Use the EXIF orientation of photos taken by smartphones.
         img = ImageOps.exif_transpose(img)
+
+        if to_scale:
+            p.width = int(img.width * scale_by)
+            p.height = int(img.height * scale_by)
+
         p.init_images = [img] * p.batch_size
 
+        image_path = Path(image)
+        if is_inpaint_batch:
+            # try to find corresponding mask for an image using simple filename matching
+            if len(inpaint_masks) == 1:
+                mask_image_path = inpaint_masks[0]
+            else:
+                # try to find corresponding mask for an image using simple filename matching
+                mask_image_dir = Path(inpaint_mask_dir)
+                masks_found = list(mask_image_dir.glob(f"{image_path.stem}.*"))
+
+                if len(masks_found) == 0:
+                    print(f"Warning: mask is not found for {image_path} in {mask_image_dir}. Skipping it.")
+                    continue
+
+                # it should contain only 1 matching mask
+                # otherwise user has many masks with the same name but different extensions
+                mask_image_path = masks_found[0]
+
+            mask_image = Image.open(mask_image_path)
+            p.image_mask = mask_image
+
         proc = modules.scripts.scripts_img2img.run(p, *args)
         if proc is None:
             proc = process_images(p)
 
         for n, processed_image in enumerate(proc.images):
-            filename = os.path.basename(image)
+            filename = image_path.name
 
             if n > 0:
                 left, right = os.path.splitext(filename)
@@ -56,10 +92,14 @@ def process_batch(p, input_dir, output_dir, args):
 
             if not save_normally:
                 os.makedirs(output_dir, exist_ok=True)
+                if processed_image.mode == 'RGBA':
+                    processed_image = processed_image.convert("RGB")
                 processed_image.save(os.path.join(output_dir, filename))
 
 
-def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, height: int, width: int, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, *args):
+def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_styles, init_img, sketch, init_img_with_mask, inpaint_color_sketch, inpaint_color_sketch_orig, init_img_inpaint, init_mask_inpaint, steps: int, sampler_index: int, mask_blur: int, mask_alpha: float, inpainting_fill: int, restore_faces: bool, tiling: bool, n_iter: int, batch_size: int, cfg_scale: float, image_cfg_scale: float, denoising_strength: float, seed: int, subseed: int, subseed_strength: float, seed_resize_from_h: int, seed_resize_from_w: int, seed_enable_extras: bool, selected_scale_tab: int, height: int, width: int, scale_by: float, resize_mode: int, inpaint_full_res: bool, inpaint_full_res_padding: int, inpainting_mask_invert: int, img2img_batch_input_dir: str, img2img_batch_output_dir: str, img2img_batch_inpaint_mask_dir: str, override_settings_texts, *args):
+    override_settings = create_override_settings_dict(override_settings_texts)
+
     is_batch = mode == 5
 
     if mode == 0:  # img2img
@@ -71,7 +111,8 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
     elif mode == 2:  # inpaint
         image, mask = init_img_with_mask["image"], init_img_with_mask["mask"]
         alpha_mask = ImageOps.invert(image.split()[-1]).convert('L').point(lambda x: 255 if x > 0 else 0, mode='1')
-        mask = ImageChops.lighter(alpha_mask, mask.convert('L')).convert('L')
+        mask = mask.convert('L').point(lambda x: 255 if x > 128 else 0, mode='1')
+        mask = ImageChops.lighter(alpha_mask, mask).convert('L')
         image = image.convert("RGB")
     elif mode == 3:  # inpaint sketch
         image = inpaint_color_sketch
@@ -93,6 +134,12 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
     if image is not None:
         image = ImageOps.exif_transpose(image)
 
+    if selected_scale_tab == 1 and not is_batch:
+        assert image, "Can't scale by because no image is selected"
+
+        width = int(image.width * scale_by)
+        height = int(image.height * scale_by)
+
     assert 0. <= denoising_strength <= 1., 'can only work with strength in [0.0, 1.0]'
 
     p = StableDiffusionProcessingImg2Img(
@@ -123,23 +170,26 @@ def img2img(id_task: str, mode: int, prompt: str, negative_prompt: str, prompt_s
         inpainting_fill=inpainting_fill,
         resize_mode=resize_mode,
         denoising_strength=denoising_strength,
+        image_cfg_scale=image_cfg_scale,
         inpaint_full_res=inpaint_full_res,
         inpaint_full_res_padding=inpaint_full_res_padding,
         inpainting_mask_invert=inpainting_mask_invert,
+        override_settings=override_settings,
     )
 
-    p.scripts = modules.scripts.scripts_txt2img
+    p.scripts = modules.scripts.scripts_img2img
     p.script_args = args
 
     if shared.cmd_opts.enable_console_prompts:
         print(f"\nimg2img: {prompt}", file=shared.progress_print_out)
 
-    p.extra_generation_params["Mask blur"] = mask_blur
+    if mask:
+        p.extra_generation_params["Mask blur"] = mask_blur
 
     if is_batch:
         assert not shared.cmd_opts.hide_ui_dir_config, "Launched with --hide-ui-dir-config, batch img2img disabled"
 
-        process_batch(p, img2img_batch_input_dir, img2img_batch_output_dir, args)
+        process_batch(p, img2img_batch_input_dir, img2img_batch_output_dir, img2img_batch_inpaint_mask_dir, args, to_scale=selected_scale_tab == 1, scale_by=scale_by)
 
         processed = Processed(p, [], p.seed, "")
     else:

modules/interrogate.py

@@ -1,6 +1,5 @@
 import os
 import sys
-import traceback
 from collections import namedtuple
 from pathlib import Path
 import re
@@ -11,8 +10,7 @@ import torch.hub
 from torchvision import transforms
 from torchvision.transforms.functional import InterpolationMode
 
-import modules.shared as shared
-from modules import devices, paths, lowvram, modelloader, errors
+from modules import devices, paths, shared, lowvram, modelloader, errors
 
 blip_image_eval_size = 384
 clip_model_name = 'ViT-L/14'
@@ -28,11 +26,11 @@ def category_types():
 def download_default_clip_interrogate_categories(content_dir):
     print("Downloading CLIP categories...")
 
-    tmpdir = content_dir + "_tmp"
+    tmpdir = f"{content_dir}_tmp"
     category_types = ["artists", "flavors", "mediums", "movements"]
 
     try:
-        os.makedirs(tmpdir)
+        os.makedirs(tmpdir, exist_ok=True)
         for category_type in category_types:
             torch.hub.download_url_to_file(f"https://raw.githubusercontent.com/pharmapsychotic/clip-interrogator/main/clip_interrogator/data/{category_type}.txt", os.path.join(tmpdir, f"{category_type}.txt"))
         os.rename(tmpdir, content_dir)
@@ -41,7 +39,7 @@ def download_default_clip_interrogate_categories(content_dir):
         errors.display(e, "downloading default CLIP interrogate categories")
     finally:
         if os.path.exists(tmpdir):
-            os.remove(tmpdir)
+            os.removedirs(tmpdir)
 
 
 class InterrogateModels:
@@ -160,7 +158,7 @@ class InterrogateModels:
             text_array = text_array[0:int(shared.opts.interrogate_clip_dict_limit)]
 
         top_count = min(top_count, len(text_array))
-        text_tokens = clip.tokenize([text for text in text_array], truncate=True).to(devices.device_interrogate)
+        text_tokens = clip.tokenize(list(text_array), truncate=True).to(devices.device_interrogate)
         text_features = self.clip_model.encode_text(text_tokens).type(self.dtype)
         text_features /= text_features.norm(dim=-1, keepdim=True)
 
@@ -208,17 +206,16 @@ class InterrogateModels:
 
                 image_features /= image_features.norm(dim=-1, keepdim=True)
 
-                for name, topn, items in self.categories():
-                    matches = self.rank(image_features, items, top_count=topn)
+                for cat in self.categories():
+                    matches = self.rank(image_features, cat.items, top_count=cat.topn)
                     for match, score in matches:
                         if shared.opts.interrogate_return_ranks:
                             res += f", ({match}:{score/100:.3f})"
                         else:
-                            res += ", " + match
+                            res += f", {match}"
 
         except Exception:
-            print("Error interrogating", file=sys.stderr)
-            print(traceback.format_exc(), file=sys.stderr)
+            errors.report("Error interrogating", exc_info=True)
             res += "<error>"
 
         self.unload()

modules/launch_utils.py

@@ -0,0 +1,344 @@
+# this scripts installs necessary requirements and launches main program in webui.py
+import subprocess
+import os
+import sys
+import importlib.util
+import platform
+import json
+from functools import lru_cache
+
+from modules import cmd_args, errors
+from modules.paths_internal import script_path, extensions_dir
+
+args, _ = cmd_args.parser.parse_known_args()
+
+python = sys.executable
+git = os.environ.get('GIT', "git")
+index_url = os.environ.get('INDEX_URL', "")
+dir_repos = "repositories"
+
+# Whether to default to printing command output
+default_command_live = (os.environ.get('WEBUI_LAUNCH_LIVE_OUTPUT') == "1")
+
+if 'GRADIO_ANALYTICS_ENABLED' not in os.environ:
+    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+
+
+def check_python_version():
+    is_windows = platform.system() == "Windows"
+    major = sys.version_info.major
+    minor = sys.version_info.minor
+    micro = sys.version_info.micro
+
+    if is_windows:
+        supported_minors = [10]
+    else:
+        supported_minors = [7, 8, 9, 10, 11]
+
+    if not (major == 3 and minor in supported_minors):
+        import modules.errors
+
+        modules.errors.print_error_explanation(f"""
+INCOMPATIBLE PYTHON VERSION
+
+This program is tested with 3.10.6 Python, but you have {major}.{minor}.{micro}.
+If you encounter an error with "RuntimeError: Couldn't install torch." message,
+or any other error regarding unsuccessful package (library) installation,
+please downgrade (or upgrade) to the latest version of 3.10 Python
+and delete current Python and "venv" folder in WebUI's directory.
+
+You can download 3.10 Python from here: https://www.python.org/downloads/release/python-3106/
+
+{"Alternatively, use a binary release of WebUI: https://github.com/AUTOMATIC1111/stable-diffusion-webui/releases" if is_windows else ""}
+
+Use --skip-python-version-check to suppress this warning.
+""")
+
+
+@lru_cache()
+def commit_hash():
+    try:
+        return subprocess.check_output([git, "rev-parse", "HEAD"], shell=False, encoding='utf8').strip()
+    except Exception:
+        return "<none>"
+
+
+@lru_cache()
+def git_tag():
+    try:
+        return subprocess.check_output([git, "describe", "--tags"], shell=False, encoding='utf8').strip()
+    except Exception:
+        try:
+            from pathlib import Path
+            changelog_md = Path(__file__).parent.parent / "CHANGELOG.md"
+            with changelog_md.open(encoding="utf-8") as file:
+                return next((line.strip() for line in file if line.strip()), "<none>")
+        except Exception:
+            return "<none>"
+
+
+def run(command, desc=None, errdesc=None, custom_env=None, live: bool = default_command_live) -> str:
+    if desc is not None:
+        print(desc)
+
+    run_kwargs = {
+        "args": command,
+        "shell": True,
+        "env": os.environ if custom_env is None else custom_env,
+        "encoding": 'utf8',
+        "errors": 'ignore',
+    }
+
+    if not live:
+        run_kwargs["stdout"] = run_kwargs["stderr"] = subprocess.PIPE
+
+    result = subprocess.run(**run_kwargs)
+
+    if result.returncode != 0:
+        error_bits = [
+            f"{errdesc or 'Error running command'}.",
+            f"Command: {command}",
+            f"Error code: {result.returncode}",
+        ]
+        if result.stdout:
+            error_bits.append(f"stdout: {result.stdout}")
+        if result.stderr:
+            error_bits.append(f"stderr: {result.stderr}")
+        raise RuntimeError("\n".join(error_bits))
+
+    return (result.stdout or "")
+
+
+def is_installed(package):
+    try:
+        spec = importlib.util.find_spec(package)
+    except ModuleNotFoundError:
+        return False
+
+    return spec is not None
+
+
+def repo_dir(name):
+    return os.path.join(script_path, dir_repos, name)
+
+
+def run_pip(command, desc=None, live=default_command_live):
+    if args.skip_install:
+        return
+
+    index_url_line = f' --index-url {index_url}' if index_url != '' else ''
+    return run(f'"{python}" -m pip {command} --prefer-binary{index_url_line}', desc=f"Installing {desc}", errdesc=f"Couldn't install {desc}", live=live)
+
+
+def check_run_python(code: str) -> bool:
+    result = subprocess.run([python, "-c", code], capture_output=True, shell=False)
+    return result.returncode == 0
+
+
+def git_clone(url, dir, name, commithash=None):
+    # TODO clone into temporary dir and move if successful
+
+    if os.path.exists(dir):
+        if commithash is None:
+            return
+
+        current_hash = run(f'"{git}" -C "{dir}" rev-parse HEAD', None, f"Couldn't determine {name}'s hash: {commithash}").strip()
+        if current_hash == commithash:
+            return
+
+        run(f'"{git}" -C "{dir}" fetch', f"Fetching updates for {name}...", f"Couldn't fetch {name}")
+        run(f'"{git}" -C "{dir}" checkout {commithash}', f"Checking out commit for {name} with hash: {commithash}...", f"Couldn't checkout commit {commithash} for {name}")
+        return
+
+    run(f'"{git}" clone "{url}" "{dir}"', f"Cloning {name} into {dir}...", f"Couldn't clone {name}")
+
+    if commithash is not None:
+        run(f'"{git}" -C "{dir}" checkout {commithash}', None, "Couldn't checkout {name}'s hash: {commithash}")
+
+
+def git_pull_recursive(dir):
+    for subdir, _, _ in os.walk(dir):
+        if os.path.exists(os.path.join(subdir, '.git')):
+            try:
+                output = subprocess.check_output([git, '-C', subdir, 'pull', '--autostash'])
+                print(f"Pulled changes for repository in '{subdir}':\n{output.decode('utf-8').strip()}\n")
+            except subprocess.CalledProcessError as e:
+                print(f"Couldn't perform 'git pull' on repository in '{subdir}':\n{e.output.decode('utf-8').strip()}\n")
+
+
+def version_check(commit):
+    try:
+        import requests
+        commits = requests.get('https://api.github.com/repos/AUTOMATIC1111/stable-diffusion-webui/branches/master').json()
+        if commit != "<none>" and commits['commit']['sha'] != commit:
+            print("--------------------------------------------------------")
+            print("| You are not up to date with the most recent release. |")
+            print("| Consider running `git pull` to update.               |")
+            print("--------------------------------------------------------")
+        elif commits['commit']['sha'] == commit:
+            print("You are up to date with the most recent release.")
+        else:
+            print("Not a git clone, can't perform version check.")
+    except Exception as e:
+        print("version check failed", e)
+
+
+def run_extension_installer(extension_dir):
+    path_installer = os.path.join(extension_dir, "install.py")
+    if not os.path.isfile(path_installer):
+        return
+
+    try:
+        env = os.environ.copy()
+        env['PYTHONPATH'] = os.path.abspath(".")
+
+        print(run(f'"{python}" "{path_installer}"', errdesc=f"Error running install.py for extension {extension_dir}", custom_env=env))
+    except Exception as e:
+        errors.report(str(e))
+
+
+def list_extensions(settings_file):
+    settings = {}
+
+    try:
+        if os.path.isfile(settings_file):
+            with open(settings_file, "r", encoding="utf8") as file:
+                settings = json.load(file)
+    except Exception:
+        errors.report("Could not load settings", exc_info=True)
+
+    disabled_extensions = set(settings.get('disabled_extensions', []))
+    disable_all_extensions = settings.get('disable_all_extensions', 'none')
+
+    if disable_all_extensions != 'none':
+        return []
+
+    return [x for x in os.listdir(extensions_dir) if x not in disabled_extensions]
+
+
+def run_extensions_installers(settings_file):
+    if not os.path.isdir(extensions_dir):
+        return
+
+    for dirname_extension in list_extensions(settings_file):
+        run_extension_installer(os.path.join(extensions_dir, dirname_extension))
+
+
+def prepare_environment():
+    torch_index_url = os.environ.get('TORCH_INDEX_URL', "https://download.pytorch.org/whl/cu118")
+    torch_command = os.environ.get('TORCH_COMMAND', f"pip install torch==2.0.1 torchvision==0.15.2 --extra-index-url {torch_index_url}")
+    requirements_file = os.environ.get('REQS_FILE', "requirements_versions.txt")
+
+    xformers_package = os.environ.get('XFORMERS_PACKAGE', 'xformers==0.0.20')
+    gfpgan_package = os.environ.get('GFPGAN_PACKAGE', "https://github.com/TencentARC/GFPGAN/archive/8d2447a2d918f8eba5a4a01463fd48e45126a379.zip")
+    clip_package = os.environ.get('CLIP_PACKAGE', "https://github.com/openai/CLIP/archive/d50d76daa670286dd6cacf3bcd80b5e4823fc8e1.zip")
+    openclip_package = os.environ.get('OPENCLIP_PACKAGE', "https://github.com/mlfoundations/open_clip/archive/bb6e834e9c70d9c27d0dc3ecedeebeaeb1ffad6b.zip")
+
+    stable_diffusion_repo = os.environ.get('STABLE_DIFFUSION_REPO', "https://github.com/Stability-AI/stablediffusion.git")
+    k_diffusion_repo = os.environ.get('K_DIFFUSION_REPO', 'https://github.com/crowsonkb/k-diffusion.git')
+    codeformer_repo = os.environ.get('CODEFORMER_REPO', 'https://github.com/sczhou/CodeFormer.git')
+    blip_repo = os.environ.get('BLIP_REPO', 'https://github.com/salesforce/BLIP.git')
+
+    stable_diffusion_commit_hash = os.environ.get('STABLE_DIFFUSION_COMMIT_HASH', "cf1d67a6fd5ea1aa600c4df58e5b47da45f6bdbf")
+    k_diffusion_commit_hash = os.environ.get('K_DIFFUSION_COMMIT_HASH', "c9fe758757e022f05ca5a53fa8fac28889e4f1cf")
+    codeformer_commit_hash = os.environ.get('CODEFORMER_COMMIT_HASH', "c5b4593074ba6214284d6acd5f1719b6c5d739af")
+    blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9")
+
+    try:
+        # the existance of this file is a signal to webui.sh/bat that webui needs to be restarted when it stops execution
+        os.remove(os.path.join(script_path, "tmp", "restart"))
+        os.environ.setdefault('SD_WEBUI_RESTARTING ', '1')
+    except OSError:
+        pass
+
+    if not args.skip_python_version_check:
+        check_python_version()
+
+    commit = commit_hash()
+    tag = git_tag()
+
+    print(f"Python {sys.version}")
+    print(f"Version: {tag}")
+    print(f"Commit hash: {commit}")
+
+    if args.reinstall_torch or not is_installed("torch") or not is_installed("torchvision"):
+        run(f'"{python}" -m {torch_command}', "Installing torch and torchvision", "Couldn't install torch", live=True)
+
+    if not args.skip_torch_cuda_test and not check_run_python("import torch; assert torch.cuda.is_available()"):
+        raise RuntimeError(
+            'Torch is not able to use GPU; '
+            'add --skip-torch-cuda-test to COMMANDLINE_ARGS variable to disable this check'
+        )
+
+    if not is_installed("gfpgan"):
+        run_pip(f"install {gfpgan_package}", "gfpgan")
+
+    if not is_installed("clip"):
+        run_pip(f"install {clip_package}", "clip")
+
+    if not is_installed("open_clip"):
+        run_pip(f"install {openclip_package}", "open_clip")
+
+    if (not is_installed("xformers") or args.reinstall_xformers) and args.xformers:
+        if platform.system() == "Windows":
+            if platform.python_version().startswith("3.10"):
+                run_pip(f"install -U -I --no-deps {xformers_package}", "xformers", live=True)
+            else:
+                print("Installation of xformers is not supported in this version of Python.")
+                print("You can also check this and build manually: https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Xformers#building-xformers-on-windows-by-duckness")
+                if not is_installed("xformers"):
+                    exit(0)
+        elif platform.system() == "Linux":
+            run_pip(f"install -U -I --no-deps {xformers_package}", "xformers")
+
+    if not is_installed("ngrok") and args.ngrok:
+        run_pip("install ngrok", "ngrok")
+
+    os.makedirs(os.path.join(script_path, dir_repos), exist_ok=True)
+
+    git_clone(stable_diffusion_repo, repo_dir('stable-diffusion-stability-ai'), "Stable Diffusion", stable_diffusion_commit_hash)
+    git_clone(k_diffusion_repo, repo_dir('k-diffusion'), "K-diffusion", k_diffusion_commit_hash)
+    git_clone(codeformer_repo, repo_dir('CodeFormer'), "CodeFormer", codeformer_commit_hash)
+    git_clone(blip_repo, repo_dir('BLIP'), "BLIP", blip_commit_hash)
+
+    if not is_installed("lpips"):
+        run_pip(f"install -r \"{os.path.join(repo_dir('CodeFormer'), 'requirements.txt')}\"", "requirements for CodeFormer")
+
+    if not os.path.isfile(requirements_file):
+        requirements_file = os.path.join(script_path, requirements_file)
+    run_pip(f"install -r \"{requirements_file}\"", "requirements")
+
+    run_extensions_installers(settings_file=args.ui_settings_file)
+
+    if args.update_check:
+        version_check(commit)
+
+    if args.update_all_extensions:
+        git_pull_recursive(extensions_dir)
+
+    if "--exit" in sys.argv:
+        print("Exiting because of --exit argument")
+        exit(0)
+
+
+def configure_for_tests():
+    if "--api" not in sys.argv:
+        sys.argv.append("--api")
+    if "--ckpt" not in sys.argv:
+        sys.argv.append("--ckpt")
+        sys.argv.append(os.path.join(script_path, "test/test_files/empty.pt"))
+    if "--skip-torch-cuda-test" not in sys.argv:
+        sys.argv.append("--skip-torch-cuda-test")
+    if "--disable-nan-check" not in sys.argv:
+        sys.argv.append("--disable-nan-check")
+
+    os.environ['COMMANDLINE_ARGS'] = ""
+
+
+def start():
+    print(f"Launching {'API server' if '--nowebui' in sys.argv else 'Web UI'} with arguments: {' '.join(sys.argv[1:])}")
+    import webui
+    if '--nowebui' in sys.argv:
+        webui.api_only()
+    else:
+        webui.webui()

modules/localization.py

@@ -1,8 +1,7 @@
 import json
 import os
-import sys
-import traceback
 
+from modules import errors
 
 localizations = {}
 
@@ -23,7 +22,7 @@ def list_localizations(dirname):
         localizations[fn] = file.path
 
 
-def localization_js(current_localization_name):
+def localization_js(current_localization_name: str) -> str:
     fn = localizations.get(current_localization_name, None)
     data = {}
     if fn is not None:
@@ -31,7 +30,6 @@ def localization_js(current_localization_name):
             with open(fn, "r", encoding="utf8") as file:
                 data = json.load(file)
         except Exception:
-            print(f"Error loading localization from {fn}:", file=sys.stderr)
-            print(traceback.format_exc(), file=sys.stderr)
+            errors.report(f"Error loading localization from {fn}", exc_info=True)
 
-    return f"var localization = {json.dumps(data)}\n"
+    return f"window.localization = {json.dumps(data)}"

modules/lowvram.py

@@ -15,6 +15,8 @@ def send_everything_to_cpu():
 
 
 def setup_for_low_vram(sd_model, use_medvram):
+    sd_model.lowvram = True
+
     parents = {}
 
     def send_me_to_gpu(module, _):
@@ -55,12 +57,12 @@ def setup_for_low_vram(sd_model, use_medvram):
     if hasattr(sd_model.cond_stage_model, 'model'):
         sd_model.cond_stage_model.transformer = sd_model.cond_stage_model.model
 
-    # remove four big modules, cond, first_stage, depth (if applicable), and unet from the model and then
+    # remove several big modules: cond, first_stage, depth/embedder (if applicable), and unet from the model and then
     # send the model to GPU. Then put modules back. the modules will be in CPU.
-    stored = sd_model.cond_stage_model.transformer, sd_model.first_stage_model, getattr(sd_model, 'depth_model', None), sd_model.model
-    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.model = None, None, None, None
+    stored = sd_model.cond_stage_model.transformer, sd_model.first_stage_model, getattr(sd_model, 'depth_model', None), getattr(sd_model, 'embedder', None), sd_model.model
+    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.embedder, sd_model.model = None, None, None, None, None
     sd_model.to(devices.device)
-    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.model = stored
+    sd_model.cond_stage_model.transformer, sd_model.first_stage_model, sd_model.depth_model, sd_model.embedder, sd_model.model = stored
 
     # register hooks for those the first three models
     sd_model.cond_stage_model.transformer.register_forward_pre_hook(send_me_to_gpu)
@@ -69,6 +71,8 @@ def setup_for_low_vram(sd_model, use_medvram):
     sd_model.first_stage_model.decode = first_stage_model_decode_wrap
     if sd_model.depth_model:
         sd_model.depth_model.register_forward_pre_hook(send_me_to_gpu)
+    if sd_model.embedder:
+        sd_model.embedder.register_forward_pre_hook(send_me_to_gpu)
     parents[sd_model.cond_stage_model.transformer] = sd_model.cond_stage_model
 
     if hasattr(sd_model.cond_stage_model, 'model'):
@@ -94,3 +98,7 @@ def setup_for_low_vram(sd_model, use_medvram):
         diff_model.middle_block.register_forward_pre_hook(send_me_to_gpu)
         for block in diff_model.output_blocks:
             block.register_forward_pre_hook(send_me_to_gpu)
+
+
+def is_enabled(sd_model):
+    return getattr(sd_model, 'lowvram', False)

modules/mac_specific.py

@@ -0,0 +1,63 @@
+import torch
+import platform
+from modules.sd_hijack_utils import CondFunc
+from packaging import version
+
+
+# has_mps is only available in nightly pytorch (for now) and macOS 12.3+.
+# check `getattr` and try it for compatibility
+def check_for_mps() -> bool:
+    if not getattr(torch, 'has_mps', False):
+        return False
+    try:
+        torch.zeros(1).to(torch.device("mps"))
+        return True
+    except Exception:
+        return False
+has_mps = check_for_mps()
+
+
+# MPS workaround for https://github.com/pytorch/pytorch/issues/89784
+def cumsum_fix(input, cumsum_func, *args, **kwargs):
+    if input.device.type == 'mps':
+        output_dtype = kwargs.get('dtype', input.dtype)
+        if output_dtype == torch.int64:
+            return cumsum_func(input.cpu(), *args, **kwargs).to(input.device)
+        elif output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16):
+            return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64)
+    return cumsum_func(input, *args, **kwargs)
+
+
+if has_mps:
+    # MPS fix for randn in torchsde
+    CondFunc('torchsde._brownian.brownian_interval._randn', lambda _, size, dtype, device, seed: torch.randn(size, dtype=dtype, device=torch.device("cpu"), generator=torch.Generator(torch.device("cpu")).manual_seed(int(seed))).to(device), lambda _, size, dtype, device, seed: device.type == 'mps')
+
+    if platform.mac_ver()[0].startswith("13.2."):
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/95188, thanks to danieldk (https://github.com/explosion/curated-transformers/pull/124)
+        CondFunc('torch.nn.functional.linear', lambda _, input, weight, bias: (torch.matmul(input, weight.t()) + bias) if bias is not None else torch.matmul(input, weight.t()), lambda _, input, weight, bias: input.numel() > 10485760)
+
+    if version.parse(torch.__version__) < version.parse("1.13"):
+        # PyTorch 1.13 doesn't need these fixes but unfortunately is slower and has regressions that prevent training from working
+
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/79383
+        CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs),
+                                                          lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps'))
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/80800
+        CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs),
+                                                                                        lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps')
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/90532
+        CondFunc('torch.Tensor.numpy', lambda orig_func, self, *args, **kwargs: orig_func(self.detach(), *args, **kwargs), lambda _, self, *args, **kwargs: self.requires_grad)
+    elif version.parse(torch.__version__) > version.parse("1.13.1"):
+        cumsum_needs_int_fix = not torch.Tensor([1,2]).to(torch.device("mps")).equal(torch.ShortTensor([1,1]).to(torch.device("mps")).cumsum(0))
+        cumsum_fix_func = lambda orig_func, input, *args, **kwargs: cumsum_fix(input, orig_func, *args, **kwargs)
+        CondFunc('torch.cumsum', cumsum_fix_func, None)
+        CondFunc('torch.Tensor.cumsum', cumsum_fix_func, None)
+        CondFunc('torch.narrow', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).clone(), None)
+
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/96113
+        CondFunc('torch.nn.functional.layer_norm', lambda orig_func, x, normalized_shape, weight, bias, eps, **kwargs: orig_func(x.float(), normalized_shape, weight.float() if weight is not None else None, bias.float() if bias is not None else bias, eps).to(x.dtype), lambda _, input, *args, **kwargs: len(args) == 4 and input.device.type == 'mps')
+
+        # MPS workaround for https://github.com/pytorch/pytorch/issues/92311
+        if platform.processor() == 'i386':
+            for funcName in ['torch.argmax', 'torch.Tensor.argmax']:
+                CondFunc(funcName, lambda _, input, *args, **kwargs: torch.max(input.float() if input.dtype == torch.int64 else input, *args, **kwargs)[1], lambda _, input, *args, **kwargs: input.device.type == 'mps')

modules/masking.py

@@ -4,7 +4,7 @@ from PIL import Image, ImageFilter, ImageOps
 def get_crop_region(mask, pad=0):
     """finds a rectangular region that contains all masked ares in an image. Returns (x1, y1, x2, y2) coordinates of the rectangle.
     For example, if a user has painted the top-right part of a 512x512 image", the result may be (256, 0, 512, 256)"""
-    
+
     h, w = mask.shape
 
     crop_left = 0

modules/memmon.py

@@ -23,12 +23,16 @@ class MemUsageMonitor(threading.Thread):
         self.data = defaultdict(int)
 
         try:
-            torch.cuda.mem_get_info()
+            self.cuda_mem_get_info()
             torch.cuda.memory_stats(self.device)
         except Exception as e:  # AMD or whatever
             print(f"Warning: caught exception '{e}', memory monitor disabled")
             self.disabled = True
 
+    def cuda_mem_get_info(self):
+        index = self.device.index if self.device.index is not None else torch.cuda.current_device()
+        return torch.cuda.mem_get_info(index)
+
     def run(self):
         if self.disabled:
             return
@@ -43,10 +47,10 @@ class MemUsageMonitor(threading.Thread):
                 self.run_flag.clear()
                 continue
 
-            self.data["min_free"] = torch.cuda.mem_get_info()[0]
+            self.data["min_free"] = self.cuda_mem_get_info()[0]
 
             while self.run_flag.is_set():
-                free, total = torch.cuda.mem_get_info()  # calling with self.device errors, torch bug?
+                free, total = self.cuda_mem_get_info()
                 self.data["min_free"] = min(self.data["min_free"], free)
 
                 time.sleep(1 / self.opts.memmon_poll_rate)
@@ -70,7 +74,7 @@ class MemUsageMonitor(threading.Thread):
 
     def read(self):
         if not self.disabled:
-            free, total = torch.cuda.mem_get_info()
+            free, total = self.cuda_mem_get_info()
             self.data["free"] = free
             self.data["total"] = total
 

modules/modelloader.py

@@ -1,12 +1,10 @@
-import glob
 import os
 import shutil
 import importlib
 from urllib.parse import urlparse
 
-from basicsr.utils.download_util import load_file_from_url
 from modules import shared
-from modules.upscaler import Upscaler
+from modules.upscaler import Upscaler, UpscalerLanczos, UpscalerNearest, UpscalerNone
 from modules.paths import script_path, models_path
 
 
@@ -23,9 +21,6 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
     """
     output = []
 
-    if ext_filter is None:
-        ext_filter = []
-
     try:
         places = []
 
@@ -40,23 +35,19 @@ def load_models(model_path: str, model_url: str = None, command_path: str = None
         places.append(model_path)
 
         for place in places:
-            if os.path.exists(place):
-                for file in glob.iglob(place + '**/**', recursive=True):
-                    full_path = file
-                    if os.path.isdir(full_path):
-                        continue
-                    if ext_blacklist is not None and any([full_path.endswith(x) for x in ext_blacklist]):
-                        continue
-                    if len(ext_filter) != 0:
-                        model_name, extension = os.path.splitext(file)
-                        if extension not in ext_filter:
-                            continue
-                    if file not in output:
-                        output.append(full_path)
+            for full_path in shared.walk_files(place, allowed_extensions=ext_filter):
+                if os.path.islink(full_path) and not os.path.exists(full_path):
+                    print(f"Skipping broken symlink: {full_path}")
+                    continue
+                if ext_blacklist is not None and any(full_path.endswith(x) for x in ext_blacklist):
+                    continue
+                if full_path not in output:
+                    output.append(full_path)
 
         if model_url is not None and len(output) == 0:
             if download_name is not None:
-                dl = load_file_from_url(model_url, model_path, True, download_name)
+                from basicsr.utils.download_util import load_file_from_url
+                dl = load_file_from_url(model_url, places[0], True, download_name)
                 output.append(dl)
             else:
                 output.append(model_url)
@@ -116,32 +107,15 @@ def move_files(src_path: str, dest_path: str, ext_filter: str = None):
                     print(f"Moving {file} from {src_path} to {dest_path}.")
                     try:
                         shutil.move(fullpath, dest_path)
-                    except:
+                    except Exception:
                         pass
             if len(os.listdir(src_path)) == 0:
                 print(f"Removing empty folder: {src_path}")
                 shutil.rmtree(src_path, True)
-    except:
+    except Exception:
         pass
 
 
-builtin_upscaler_classes = []
-forbidden_upscaler_classes = set()
-
-
-def list_builtin_upscalers():
-    load_upscalers()
-
-    builtin_upscaler_classes.clear()
-    builtin_upscaler_classes.extend(Upscaler.__subclasses__())
-
-
-def forbid_loaded_nonbuiltin_upscalers():
-    for cls in Upscaler.__subclasses__():
-        if cls not in builtin_upscaler_classes:
-            forbidden_upscaler_classes.add(cls)
-
-
 def load_upscalers():
     # We can only do this 'magic' method to dynamically load upscalers if they are referenced,
     # so we'll try to import any _model.py files before looking in __subclasses__
@@ -152,18 +126,32 @@ def load_upscalers():
             full_model = f"modules.{model_name}_model"
             try:
                 importlib.import_module(full_model)
-            except:
+            except Exception:
                 pass
 
     datas = []
     commandline_options = vars(shared.cmd_opts)
-    for cls in Upscaler.__subclasses__():
-        if cls in forbidden_upscaler_classes:
-            continue
 
+    # some of upscaler classes will not go away after reloading their modules, and we'll end
+    # up with two copies of those classes. The newest copy will always be the last in the list,
+    # so we go from end to beginning and ignore duplicates
+    used_classes = {}
+    for cls in reversed(Upscaler.__subclasses__()):
+        classname = str(cls)
+        if classname not in used_classes:
+            used_classes[classname] = cls
+
+    for cls in reversed(used_classes.values()):
         name = cls.__name__
         cmd_name = f"{name.lower().replace('upscaler', '')}_models_path"
-        scaler = cls(commandline_options.get(cmd_name, None))
+        commandline_model_path = commandline_options.get(cmd_name, None)
+        scaler = cls(commandline_model_path)
+        scaler.user_path = commandline_model_path
+        scaler.model_download_path = commandline_model_path or scaler.model_path
         datas += scaler.scalers
 
-    shared.sd_upscalers = datas
+    shared.sd_upscalers = sorted(
+        datas,
+        # Special case for UpscalerNone keeps it at the beginning of the list.
+        key=lambda x: x.name.lower() if not isinstance(x.scaler, (UpscalerNone, UpscalerLanczos, UpscalerNearest)) else ""
+    )

modules/models/diffusion/uni_pc/__init__.py

@@ -0,0 +1 @@
+from .sampler import UniPCSampler  # noqa: F401

modules/models/diffusion/uni_pc/sampler.py

@@ -0,0 +1,101 @@
+"""SAMPLING ONLY."""
+
+import torch
+
+from .uni_pc import NoiseScheduleVP, model_wrapper, UniPC
+from modules import shared, devices
+
+
+class UniPCSampler(object):
+    def __init__(self, model, **kwargs):
+        super().__init__()
+        self.model = model
+        to_torch = lambda x: x.clone().detach().to(torch.float32).to(model.device)
+        self.before_sample = None
+        self.after_sample = None
+        self.register_buffer('alphas_cumprod', to_torch(model.alphas_cumprod))
+
+    def register_buffer(self, name, attr):
+        if type(attr) == torch.Tensor:
+            if attr.device != devices.device:
+                attr = attr.to(devices.device)
+        setattr(self, name, attr)
+
+    def set_hooks(self, before_sample, after_sample, after_update):
+        self.before_sample = before_sample
+        self.after_sample = after_sample
+        self.after_update = after_update
+
+    @torch.no_grad()
+    def sample(self,
+               S,
+               batch_size,
+               shape,
+               conditioning=None,
+               callback=None,
+               normals_sequence=None,
+               img_callback=None,
+               quantize_x0=False,
+               eta=0.,
+               mask=None,
+               x0=None,
+               temperature=1.,
+               noise_dropout=0.,
+               score_corrector=None,
+               corrector_kwargs=None,
+               verbose=True,
+               x_T=None,
+               log_every_t=100,
+               unconditional_guidance_scale=1.,
+               unconditional_conditioning=None,
+               # this has to come in the same format as the conditioning, # e.g. as encoded tokens, ...
+               **kwargs
+               ):
+        if conditioning is not None:
+            if isinstance(conditioning, dict):
+                ctmp = conditioning[list(conditioning.keys())[0]]
+                while isinstance(ctmp, list):
+                    ctmp = ctmp[0]
+                cbs = ctmp.shape[0]
+                if cbs != batch_size:
+                    print(f"Warning: Got {cbs} conditionings but batch-size is {batch_size}")
+
+            elif isinstance(conditioning, list):
+                for ctmp in conditioning:
+                    if ctmp.shape[0] != batch_size:
+                        print(f"Warning: Got {cbs} conditionings but batch-size is {batch_size}")
+
+            else:
+                if conditioning.shape[0] != batch_size:
+                    print(f"Warning: Got {conditioning.shape[0]} conditionings but batch-size is {batch_size}")
+
+        # sampling
+        C, H, W = shape
+        size = (batch_size, C, H, W)
+        # print(f'Data shape for UniPC sampling is {size}')
+
+        device = self.model.betas.device
+        if x_T is None:
+            img = torch.randn(size, device=device)
+        else:
+            img = x_T
+
+        ns = NoiseScheduleVP('discrete', alphas_cumprod=self.alphas_cumprod)
+
+        # SD 1.X is "noise", SD 2.X is "v"
+        model_type = "v" if self.model.parameterization == "v" else "noise"
+
+        model_fn = model_wrapper(
+            lambda x, t, c: self.model.apply_model(x, t, c),
+            ns,
+            model_type=model_type,
+            guidance_type="classifier-free",
+            #condition=conditioning,
+            #unconditional_condition=unconditional_conditioning,
+            guidance_scale=unconditional_guidance_scale,
+        )
+
+        uni_pc = UniPC(model_fn, ns, predict_x0=True, thresholding=False, variant=shared.opts.uni_pc_variant, condition=conditioning, unconditional_condition=unconditional_conditioning, before_sample=self.before_sample, after_sample=self.after_sample, after_update=self.after_update)
+        x = uni_pc.sample(img, steps=S, skip_type=shared.opts.uni_pc_skip_type, method="multistep", order=shared.opts.uni_pc_order, lower_order_final=shared.opts.uni_pc_lower_order_final)
+
+        return x.to(device), None

modules/models/diffusion/uni_pc/uni_pc.py

@@ -0,0 +1,863 @@
+import torch
+import math
+import tqdm
+
+
+class NoiseScheduleVP:
+    def __init__(
+            self,
+            schedule='discrete',
+            betas=None,
+            alphas_cumprod=None,
+            continuous_beta_0=0.1,
+            continuous_beta_1=20.,
+        ):
+        """Create a wrapper class for the forward SDE (VP type).
+
+        ***
+        Update: We support discrete-time diffusion models by implementing a picewise linear interpolation for log_alpha_t.
+                We recommend to use schedule='discrete' for the discrete-time diffusion models, especially for high-resolution images.
+        ***
+
+        The forward SDE ensures that the condition distribution q_{t|0}(x_t | x_0) = N ( alpha_t * x_0, sigma_t^2 * I ).
+        We further define lambda_t = log(alpha_t) - log(sigma_t), which is the half-logSNR (described in the DPM-Solver paper).
+        Therefore, we implement the functions for computing alpha_t, sigma_t and lambda_t. For t in [0, T], we have:
+
+            log_alpha_t = self.marginal_log_mean_coeff(t)
+            sigma_t = self.marginal_std(t)
+            lambda_t = self.marginal_lambda(t)
+
+        Moreover, as lambda(t) is an invertible function, we also support its inverse function:
+
+            t = self.inverse_lambda(lambda_t)
+
+        ===============================================================
+
+        We support both discrete-time DPMs (trained on n = 0, 1, ..., N-1) and continuous-time DPMs (trained on t in [t_0, T]).
+
+        1. For discrete-time DPMs:
+
+            For discrete-time DPMs trained on n = 0, 1, ..., N-1, we convert the discrete steps to continuous time steps by:
+                t_i = (i + 1) / N
+            e.g. for N = 1000, we have t_0 = 1e-3 and T = t_{N-1} = 1.
+            We solve the corresponding diffusion ODE from time T = 1 to time t_0 = 1e-3.
+
+            Args:
+                betas: A `torch.Tensor`. The beta array for the discrete-time DPM. (See the original DDPM paper for details)
+                alphas_cumprod: A `torch.Tensor`. The cumprod alphas for the discrete-time DPM. (See the original DDPM paper for details)
+
+            Note that we always have alphas_cumprod = cumprod(betas). Therefore, we only need to set one of `betas` and `alphas_cumprod`.
+
+            **Important**:  Please pay special attention for the args for `alphas_cumprod`:
+                The `alphas_cumprod` is the \hat{alpha_n} arrays in the notations of DDPM. Specifically, DDPMs assume that
+                    q_{t_n | 0}(x_{t_n} | x_0) = N ( \sqrt{\hat{alpha_n}} * x_0, (1 - \hat{alpha_n}) * I ).
+                Therefore, the notation \hat{alpha_n} is different from the notation alpha_t in DPM-Solver. In fact, we have
+                    alpha_{t_n} = \sqrt{\hat{alpha_n}},
+                and
+                    log(alpha_{t_n}) = 0.5 * log(\hat{alpha_n}).
+
+
+        2. For continuous-time DPMs:
+
+            We support two types of VPSDEs: linear (DDPM) and cosine (improved-DDPM). The hyperparameters for the noise
+            schedule are the default settings in DDPM and improved-DDPM:
+
+            Args:
+                beta_min: A `float` number. The smallest beta for the linear schedule.
+                beta_max: A `float` number. The largest beta for the linear schedule.
+                cosine_s: A `float` number. The hyperparameter in the cosine schedule.
+                cosine_beta_max: A `float` number. The hyperparameter in the cosine schedule.
+                T: A `float` number. The ending time of the forward process.
+
+        ===============================================================
+
+        Args:
+            schedule: A `str`. The noise schedule of the forward SDE. 'discrete' for discrete-time DPMs,
+                    'linear' or 'cosine' for continuous-time DPMs.
+        Returns:
+            A wrapper object of the forward SDE (VP type).
+
+        ===============================================================
+
+        Example:
+
+        # For discrete-time DPMs, given betas (the beta array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', betas=betas)
+
+        # For discrete-time DPMs, given alphas_cumprod (the \hat{alpha_n} array for n = 0, 1, ..., N - 1):
+        >>> ns = NoiseScheduleVP('discrete', alphas_cumprod=alphas_cumprod)
+
+        # For continuous-time DPMs (VPSDE), linear schedule:
+        >>> ns = NoiseScheduleVP('linear', continuous_beta_0=0.1, continuous_beta_1=20.)
+
+        """
+
+        if schedule not in ['discrete', 'linear', 'cosine']:
+            raise ValueError(f"Unsupported noise schedule {schedule}. The schedule needs to be 'discrete' or 'linear' or 'cosine'")
+
+        self.schedule = schedule
+        if schedule == 'discrete':
+            if betas is not None:
+                log_alphas = 0.5 * torch.log(1 - betas).cumsum(dim=0)
+            else:
+                assert alphas_cumprod is not None
+                log_alphas = 0.5 * torch.log(alphas_cumprod)
+            self.total_N = len(log_alphas)
+            self.T = 1.
+            self.t_array = torch.linspace(0., 1., self.total_N + 1)[1:].reshape((1, -1))
+            self.log_alpha_array = log_alphas.reshape((1, -1,))
+        else:
+            self.total_N = 1000
+            self.beta_0 = continuous_beta_0
+            self.beta_1 = continuous_beta_1
+            self.cosine_s = 0.008
+            self.cosine_beta_max = 999.
+            self.cosine_t_max = math.atan(self.cosine_beta_max * (1. + self.cosine_s) / math.pi) * 2. * (1. + self.cosine_s) / math.pi - self.cosine_s
+            self.cosine_log_alpha_0 = math.log(math.cos(self.cosine_s / (1. + self.cosine_s) * math.pi / 2.))
+            self.schedule = schedule
+            if schedule == 'cosine':
+                # For the cosine schedule, T = 1 will have numerical issues. So we manually set the ending time T.
+                # Note that T = 0.9946 may be not the optimal setting. However, we find it works well.
+                self.T = 0.9946
+            else:
+                self.T = 1.
+
+    def marginal_log_mean_coeff(self, t):
+        """
+        Compute log(alpha_t) of a given continuous-time label t in [0, T].
+        """
+        if self.schedule == 'discrete':
+            return interpolate_fn(t.reshape((-1, 1)), self.t_array.to(t.device), self.log_alpha_array.to(t.device)).reshape((-1))
+        elif self.schedule == 'linear':
+            return -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0
+        elif self.schedule == 'cosine':
+            log_alpha_fn = lambda s: torch.log(torch.cos((s + self.cosine_s) / (1. + self.cosine_s) * math.pi / 2.))
+            log_alpha_t =  log_alpha_fn(t) - self.cosine_log_alpha_0
+            return log_alpha_t
+
+    def marginal_alpha(self, t):
+        """
+        Compute alpha_t of a given continuous-time label t in [0, T].
+        """
+        return torch.exp(self.marginal_log_mean_coeff(t))
+
+    def marginal_std(self, t):
+        """
+        Compute sigma_t of a given continuous-time label t in [0, T].
+        """
+        return torch.sqrt(1. - torch.exp(2. * self.marginal_log_mean_coeff(t)))
+
+    def marginal_lambda(self, t):
+        """
+        Compute lambda_t = log(alpha_t) - log(sigma_t) of a given continuous-time label t in [0, T].
+        """
+        log_mean_coeff = self.marginal_log_mean_coeff(t)
+        log_std = 0.5 * torch.log(1. - torch.exp(2. * log_mean_coeff))
+        return log_mean_coeff - log_std
+
+    def inverse_lambda(self, lamb):
+        """
+        Compute the continuous-time label t in [0, T] of a given half-logSNR lambda_t.
+        """
+        if self.schedule == 'linear':
+            tmp = 2. * (self.beta_1 - self.beta_0) * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
+            Delta = self.beta_0**2 + tmp
+            return tmp / (torch.sqrt(Delta) + self.beta_0) / (self.beta_1 - self.beta_0)
+        elif self.schedule == 'discrete':
+            log_alpha = -0.5 * torch.logaddexp(torch.zeros((1,)).to(lamb.device), -2. * lamb)
+            t = interpolate_fn(log_alpha.reshape((-1, 1)), torch.flip(self.log_alpha_array.to(lamb.device), [1]), torch.flip(self.t_array.to(lamb.device), [1]))
+            return t.reshape((-1,))
+        else:
+            log_alpha = -0.5 * torch.logaddexp(-2. * lamb, torch.zeros((1,)).to(lamb))
+            t_fn = lambda log_alpha_t: torch.arccos(torch.exp(log_alpha_t + self.cosine_log_alpha_0)) * 2. * (1. + self.cosine_s) / math.pi - self.cosine_s
+            t = t_fn(log_alpha)
+            return t
+
+
+def model_wrapper(
+    model,
+    noise_schedule,
+    model_type="noise",
+    model_kwargs=None,
+    guidance_type="uncond",
+    #condition=None,
+    #unconditional_condition=None,
+    guidance_scale=1.,
+    classifier_fn=None,
+    classifier_kwargs=None,
+):
+    """Create a wrapper function for the noise prediction model.
+
+    DPM-Solver needs to solve the continuous-time diffusion ODEs. For DPMs trained on discrete-time labels, we need to
+    firstly wrap the model function to a noise prediction model that accepts the continuous time as the input.
+
+    We support four types of the diffusion model by setting `model_type`:
+
+        1. "noise": noise prediction model. (Trained by predicting noise).
+
+        2. "x_start": data prediction model. (Trained by predicting the data x_0 at time 0).
+
+        3. "v": velocity prediction model. (Trained by predicting the velocity).
+            The "v" prediction is derivation detailed in Appendix D of [1], and is used in Imagen-Video [2].
+
+            [1] Salimans, Tim, and Jonathan Ho. "Progressive distillation for fast sampling of diffusion models."
+                arXiv preprint arXiv:2202.00512 (2022).
+            [2] Ho, Jonathan, et al. "Imagen Video: High Definition Video Generation with Diffusion Models."
+                arXiv preprint arXiv:2210.02303 (2022).
+
+        4. "score": marginal score function. (Trained by denoising score matching).
+            Note that the score function and the noise prediction model follows a simple relationship:
+            ```
+                noise(x_t, t) = -sigma_t * score(x_t, t)
+            ```
+
+    We support three types of guided sampling by DPMs by setting `guidance_type`:
+        1. "uncond": unconditional sampling by DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            ``
+
+        2. "classifier": classifier guidance sampling [3] by DPMs and another classifier.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, **model_kwargs) -> noise | x_start | v | score
+            ``
+
+            The input `classifier_fn` has the following format:
+            ``
+                classifier_fn(x, t_input, cond, **classifier_kwargs) -> logits(x, t_input, cond)
+            ``
+
+            [3] P. Dhariwal and A. Q. Nichol, "Diffusion models beat GANs on image synthesis,"
+                in Advances in Neural Information Processing Systems, vol. 34, 2021, pp. 8780-8794.
+
+        3. "classifier-free": classifier-free guidance sampling by conditional DPMs.
+            The input `model` has the following format:
+            ``
+                model(x, t_input, cond, **model_kwargs) -> noise | x_start | v | score
+            ``
+            And if cond == `unconditional_condition`, the model output is the unconditional DPM output.
+
+            [4] Ho, Jonathan, and Tim Salimans. "Classifier-free diffusion guidance."
+                arXiv preprint arXiv:2207.12598 (2022).
+
+
+    The `t_input` is the time label of the model, which may be discrete-time labels (i.e. 0 to 999)
+    or continuous-time labels (i.e. epsilon to T).
+
+    We wrap the model function to accept only `x` and `t_continuous` as inputs, and outputs the predicted noise:
+    ``
+        def model_fn(x, t_continuous) -> noise:
+            t_input = get_model_input_time(t_continuous)
+            return noise_pred(model, x, t_input, **model_kwargs)
+    ``
+    where `t_continuous` is the continuous time labels (i.e. epsilon to T). And we use `model_fn` for DPM-Solver.
+
+    ===============================================================
+
+    Args:
+        model: A diffusion model with the corresponding format described above.
+        noise_schedule: A noise schedule object, such as NoiseScheduleVP.
+        model_type: A `str`. The parameterization type of the diffusion model.
+                    "noise" or "x_start" or "v" or "score".
+        model_kwargs: A `dict`. A dict for the other inputs of the model function.
+        guidance_type: A `str`. The type of the guidance for sampling.
+                    "uncond" or "classifier" or "classifier-free".
+        condition: A pytorch tensor. The condition for the guided sampling.
+                    Only used for "classifier" or "classifier-free" guidance type.
+        unconditional_condition: A pytorch tensor. The condition for the unconditional sampling.
+                    Only used for "classifier-free" guidance type.
+        guidance_scale: A `float`. The scale for the guided sampling.
+        classifier_fn: A classifier function. Only used for the classifier guidance.
+        classifier_kwargs: A `dict`. A dict for the other inputs of the classifier function.
+    Returns:
+        A noise prediction model that accepts the noised data and the continuous time as the inputs.
+    """
+
+    model_kwargs = model_kwargs or {}
+    classifier_kwargs = classifier_kwargs or {}
+
+    def get_model_input_time(t_continuous):
+        """
+        Convert the continuous-time `t_continuous` (in [epsilon, T]) to the model input time.
+        For discrete-time DPMs, we convert `t_continuous` in [1 / N, 1] to `t_input` in [0, 1000 * (N - 1) / N].
+        For continuous-time DPMs, we just use `t_continuous`.
+        """
+        if noise_schedule.schedule == 'discrete':
+            return (t_continuous - 1. / noise_schedule.total_N) * 1000.
+        else:
+            return t_continuous
+
+    def noise_pred_fn(x, t_continuous, cond=None):
+        if t_continuous.reshape((-1,)).shape[0] == 1:
+            t_continuous = t_continuous.expand((x.shape[0]))
+        t_input = get_model_input_time(t_continuous)
+        if cond is None:
+            output = model(x, t_input, None, **model_kwargs)
+        else:
+            output = model(x, t_input, cond, **model_kwargs)
+        if model_type == "noise":
+            return output
+        elif model_type == "x_start":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return (x - expand_dims(alpha_t, dims) * output) / expand_dims(sigma_t, dims)
+        elif model_type == "v":
+            alpha_t, sigma_t = noise_schedule.marginal_alpha(t_continuous), noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return expand_dims(alpha_t, dims) * output + expand_dims(sigma_t, dims) * x
+        elif model_type == "score":
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            dims = x.dim()
+            return -expand_dims(sigma_t, dims) * output
+
+    def cond_grad_fn(x, t_input, condition):
+        """
+        Compute the gradient of the classifier, i.e. nabla_{x} log p_t(cond | x_t).
+        """
+        with torch.enable_grad():
+            x_in = x.detach().requires_grad_(True)
+            log_prob = classifier_fn(x_in, t_input, condition, **classifier_kwargs)
+            return torch.autograd.grad(log_prob.sum(), x_in)[0]
+
+    def model_fn(x, t_continuous, condition, unconditional_condition):
+        """
+        The noise predicition model function that is used for DPM-Solver.
+        """
+        if t_continuous.reshape((-1,)).shape[0] == 1:
+            t_continuous = t_continuous.expand((x.shape[0]))
+        if guidance_type == "uncond":
+            return noise_pred_fn(x, t_continuous)
+        elif guidance_type == "classifier":
+            assert classifier_fn is not None
+            t_input = get_model_input_time(t_continuous)
+            cond_grad = cond_grad_fn(x, t_input, condition)
+            sigma_t = noise_schedule.marginal_std(t_continuous)
+            noise = noise_pred_fn(x, t_continuous)
+            return noise - guidance_scale * expand_dims(sigma_t, dims=cond_grad.dim()) * cond_grad
+        elif guidance_type == "classifier-free":
+            if guidance_scale == 1. or unconditional_condition is None:
+                return noise_pred_fn(x, t_continuous, cond=condition)
+            else:
+                x_in = torch.cat([x] * 2)
+                t_in = torch.cat([t_continuous] * 2)
+                if isinstance(condition, dict):
+                    assert isinstance(unconditional_condition, dict)
+                    c_in = {}
+                    for k in condition:
+                        if isinstance(condition[k], list):
+                            c_in[k] = [torch.cat([
+                                unconditional_condition[k][i],
+                                condition[k][i]]) for i in range(len(condition[k]))]
+                        else:
+                            c_in[k] = torch.cat([
+                                unconditional_condition[k],
+                                condition[k]])
+                elif isinstance(condition, list):
+                    c_in = []
+                    assert isinstance(unconditional_condition, list)
+                    for i in range(len(condition)):
+                        c_in.append(torch.cat([unconditional_condition[i], condition[i]]))
+                else:
+                    c_in = torch.cat([unconditional_condition, condition])
+                noise_uncond, noise = noise_pred_fn(x_in, t_in, cond=c_in).chunk(2)
+                return noise_uncond + guidance_scale * (noise - noise_uncond)
+
+    assert model_type in ["noise", "x_start", "v"]
+    assert guidance_type in ["uncond", "classifier", "classifier-free"]
+    return model_fn
+
+
+class UniPC:
+    def __init__(
+        self,
+        model_fn,
+        noise_schedule,
+        predict_x0=True,
+        thresholding=False,
+        max_val=1.,
+        variant='bh1',
+        condition=None,
+        unconditional_condition=None,
+        before_sample=None,
+        after_sample=None,
+        after_update=None
+    ):
+        """Construct a UniPC.
+
+        We support both data_prediction and noise_prediction.
+        """
+        self.model_fn_ = model_fn
+        self.noise_schedule = noise_schedule
+        self.variant = variant
+        self.predict_x0 = predict_x0
+        self.thresholding = thresholding
+        self.max_val = max_val
+        self.condition = condition
+        self.unconditional_condition = unconditional_condition
+        self.before_sample = before_sample
+        self.after_sample = after_sample
+        self.after_update = after_update
+
+    def dynamic_thresholding_fn(self, x0, t=None):
+        """
+        The dynamic thresholding method.
+        """
+        dims = x0.dim()
+        p = self.dynamic_thresholding_ratio
+        s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
+        s = expand_dims(torch.maximum(s, self.thresholding_max_val * torch.ones_like(s).to(s.device)), dims)
+        x0 = torch.clamp(x0, -s, s) / s
+        return x0
+
+    def model(self, x, t):
+        cond = self.condition
+        uncond = self.unconditional_condition
+        if self.before_sample is not None:
+            x, t, cond, uncond = self.before_sample(x, t, cond, uncond)
+        res = self.model_fn_(x, t, cond, uncond)
+        if self.after_sample is not None:
+            x, t, cond, uncond, res = self.after_sample(x, t, cond, uncond, res)
+
+        if isinstance(res, tuple):
+            # (None, pred_x0)
+            res = res[1]
+
+        return res
+
+    def noise_prediction_fn(self, x, t):
+        """
+        Return the noise prediction model.
+        """
+        return self.model(x, t)
+
+    def data_prediction_fn(self, x, t):
+        """
+        Return the data prediction model (with thresholding).
+        """
+        noise = self.noise_prediction_fn(x, t)
+        dims = x.dim()
+        alpha_t, sigma_t = self.noise_schedule.marginal_alpha(t), self.noise_schedule.marginal_std(t)
+        x0 = (x - expand_dims(sigma_t, dims) * noise) / expand_dims(alpha_t, dims)
+        if self.thresholding:
+            p = 0.995   # A hyperparameter in the paper of "Imagen" [1].
+            s = torch.quantile(torch.abs(x0).reshape((x0.shape[0], -1)), p, dim=1)
+            s = expand_dims(torch.maximum(s, self.max_val * torch.ones_like(s).to(s.device)), dims)
+            x0 = torch.clamp(x0, -s, s) / s
+        return x0
+
+    def model_fn(self, x, t):
+        """
+        Convert the model to the noise prediction model or the data prediction model.
+        """
+        if self.predict_x0:
+            return self.data_prediction_fn(x, t)
+        else:
+            return self.noise_prediction_fn(x, t)
+
+    def get_time_steps(self, skip_type, t_T, t_0, N, device):
+        """Compute the intermediate time steps for sampling.
+        """
+        if skip_type == 'logSNR':
+            lambda_T = self.noise_schedule.marginal_lambda(torch.tensor(t_T).to(device))
+            lambda_0 = self.noise_schedule.marginal_lambda(torch.tensor(t_0).to(device))
+            logSNR_steps = torch.linspace(lambda_T.cpu().item(), lambda_0.cpu().item(), N + 1).to(device)
+            return self.noise_schedule.inverse_lambda(logSNR_steps)
+        elif skip_type == 'time_uniform':
+            return torch.linspace(t_T, t_0, N + 1).to(device)
+        elif skip_type == 'time_quadratic':
+            t_order = 2
+            t = torch.linspace(t_T**(1. / t_order), t_0**(1. / t_order), N + 1).pow(t_order).to(device)
+            return t
+        else:
+            raise ValueError(f"Unsupported skip_type {skip_type}, need to be 'logSNR' or 'time_uniform' or 'time_quadratic'")
+
+    def get_orders_and_timesteps_for_singlestep_solver(self, steps, order, skip_type, t_T, t_0, device):
+        """
+        Get the order of each step for sampling by the singlestep DPM-Solver.
+        """
+        if order == 3:
+            K = steps // 3 + 1
+            if steps % 3 == 0:
+                orders = [3,] * (K - 2) + [2, 1]
+            elif steps % 3 == 1:
+                orders = [3,] * (K - 1) + [1]
+            else:
+                orders = [3,] * (K - 1) + [2]
+        elif order == 2:
+            if steps % 2 == 0:
+                K = steps // 2
+                orders = [2,] * K
+            else:
+                K = steps // 2 + 1
+                orders = [2,] * (K - 1) + [1]
+        elif order == 1:
+            K = steps
+            orders = [1,] * steps
+        else:
+            raise ValueError("'order' must be '1' or '2' or '3'.")
+        if skip_type == 'logSNR':
+            # To reproduce the results in DPM-Solver paper
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, K, device)
+        else:
+            timesteps_outer = self.get_time_steps(skip_type, t_T, t_0, steps, device)[torch.cumsum(torch.tensor([0,] + orders), 0).to(device)]
+        return timesteps_outer, orders
+
+    def denoise_to_zero_fn(self, x, s):
+        """
+        Denoise at the final step, which is equivalent to solve the ODE from lambda_s to infty by first-order discretization.
+        """
+        return self.data_prediction_fn(x, s)
+
+    def multistep_uni_pc_update(self, x, model_prev_list, t_prev_list, t, order, **kwargs):
+        if len(t.shape) == 0:
+            t = t.view(-1)
+        if 'bh' in self.variant:
+            return self.multistep_uni_pc_bh_update(x, model_prev_list, t_prev_list, t, order, **kwargs)
+        else:
+            assert self.variant == 'vary_coeff'
+            return self.multistep_uni_pc_vary_update(x, model_prev_list, t_prev_list, t, order, **kwargs)
+
+    def multistep_uni_pc_vary_update(self, x, model_prev_list, t_prev_list, t, order, use_corrector=True):
+        #print(f'using unified predictor-corrector with order {order} (solver type: vary coeff)')
+        ns = self.noise_schedule
+        assert order <= len(model_prev_list)
+
+        # first compute rks
+        t_prev_0 = t_prev_list[-1]
+        lambda_prev_0 = ns.marginal_lambda(t_prev_0)
+        lambda_t = ns.marginal_lambda(t)
+        model_prev_0 = model_prev_list[-1]
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        log_alpha_t = ns.marginal_log_mean_coeff(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h = lambda_t - lambda_prev_0
+
+        rks = []
+        D1s = []
+        for i in range(1, order):
+            t_prev_i = t_prev_list[-(i + 1)]
+            model_prev_i = model_prev_list[-(i + 1)]
+            lambda_prev_i = ns.marginal_lambda(t_prev_i)
+            rk = (lambda_prev_i - lambda_prev_0) / h
+            rks.append(rk)
+            D1s.append((model_prev_i - model_prev_0) / rk)
+
+        rks.append(1.)
+        rks = torch.tensor(rks, device=x.device)
+
+        K = len(rks)
+        # build C matrix
+        C = []
+
+        col = torch.ones_like(rks)
+        for k in range(1, K + 1):
+            C.append(col)
+            col = col * rks / (k + 1)
+        C = torch.stack(C, dim=1)
+
+        if len(D1s) > 0:
+            D1s = torch.stack(D1s, dim=1) # (B, K)
+            C_inv_p = torch.linalg.inv(C[:-1, :-1])
+            A_p = C_inv_p
+
+        if use_corrector:
+            #print('using corrector')
+            C_inv = torch.linalg.inv(C)
+            A_c = C_inv
+
+        hh = -h if self.predict_x0 else h
+        h_phi_1 = torch.expm1(hh)
+        h_phi_ks = []
+        factorial_k = 1
+        h_phi_k = h_phi_1
+        for k in range(1, K + 2):
+            h_phi_ks.append(h_phi_k)
+            h_phi_k = h_phi_k / hh - 1 / factorial_k
+            factorial_k *= (k + 1)
+
+        model_t = None
+        if self.predict_x0:
+            x_t_ = (
+                sigma_t / sigma_prev_0 * x
+                - alpha_t * h_phi_1 * model_prev_0
+            )
+            # now predictor
+            x_t = x_t_
+            if len(D1s) > 0:
+                # compute the residuals for predictor
+                for k in range(K - 1):
+                    x_t = x_t - alpha_t * h_phi_ks[k + 1] * torch.einsum('bkchw,k->bchw', D1s, A_p[k])
+            # now corrector
+            if use_corrector:
+                model_t = self.model_fn(x_t, t)
+                D1_t = (model_t - model_prev_0)
+                x_t = x_t_
+                k = 0
+                for k in range(K - 1):
+                    x_t = x_t - alpha_t * h_phi_ks[k + 1] * torch.einsum('bkchw,k->bchw', D1s, A_c[k][:-1])
+                x_t = x_t - alpha_t * h_phi_ks[K] * (D1_t * A_c[k][-1])
+        else:
+            log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+            x_t_ = (
+                (torch.exp(log_alpha_t - log_alpha_prev_0)) * x
+                - (sigma_t * h_phi_1) * model_prev_0
+            )
+            # now predictor
+            x_t = x_t_
+            if len(D1s) > 0:
+                # compute the residuals for predictor
+                for k in range(K - 1):
+                    x_t = x_t - sigma_t * h_phi_ks[k + 1] * torch.einsum('bkchw,k->bchw', D1s, A_p[k])
+            # now corrector
+            if use_corrector:
+                model_t = self.model_fn(x_t, t)
+                D1_t = (model_t - model_prev_0)
+                x_t = x_t_
+                k = 0
+                for k in range(K - 1):
+                    x_t = x_t - sigma_t * h_phi_ks[k + 1] * torch.einsum('bkchw,k->bchw', D1s, A_c[k][:-1])
+                x_t = x_t - sigma_t * h_phi_ks[K] * (D1_t * A_c[k][-1])
+        return x_t, model_t
+
+    def multistep_uni_pc_bh_update(self, x, model_prev_list, t_prev_list, t, order, x_t=None, use_corrector=True):
+        #print(f'using unified predictor-corrector with order {order} (solver type: B(h))')
+        ns = self.noise_schedule
+        assert order <= len(model_prev_list)
+        dims = x.dim()
+
+        # first compute rks
+        t_prev_0 = t_prev_list[-1]
+        lambda_prev_0 = ns.marginal_lambda(t_prev_0)
+        lambda_t = ns.marginal_lambda(t)
+        model_prev_0 = model_prev_list[-1]
+        sigma_prev_0, sigma_t = ns.marginal_std(t_prev_0), ns.marginal_std(t)
+        log_alpha_prev_0, log_alpha_t = ns.marginal_log_mean_coeff(t_prev_0), ns.marginal_log_mean_coeff(t)
+        alpha_t = torch.exp(log_alpha_t)
+
+        h = lambda_t - lambda_prev_0
+
+        rks = []
+        D1s = []
+        for i in range(1, order):
+            t_prev_i = t_prev_list[-(i + 1)]
+            model_prev_i = model_prev_list[-(i + 1)]
+            lambda_prev_i = ns.marginal_lambda(t_prev_i)
+            rk = ((lambda_prev_i - lambda_prev_0) / h)[0]
+            rks.append(rk)
+            D1s.append((model_prev_i - model_prev_0) / rk)
+
+        rks.append(1.)
+        rks = torch.tensor(rks, device=x.device)
+
+        R = []
+        b = []
+
+        hh = -h[0] if self.predict_x0 else h[0]
+        h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1
+        h_phi_k = h_phi_1 / hh - 1
+
+        factorial_i = 1
+
+        if self.variant == 'bh1':
+            B_h = hh
+        elif self.variant == 'bh2':
+            B_h = torch.expm1(hh)
+        else:
+            raise NotImplementedError()
+
+        for i in range(1, order + 1):
+            R.append(torch.pow(rks, i - 1))
+            b.append(h_phi_k * factorial_i / B_h)
+            factorial_i *= (i + 1)
+            h_phi_k = h_phi_k / hh - 1 / factorial_i
+
+        R = torch.stack(R)
+        b = torch.tensor(b, device=x.device)
+
+        # now predictor
+        use_predictor = len(D1s) > 0 and x_t is None
+        if len(D1s) > 0:
+            D1s = torch.stack(D1s, dim=1) # (B, K)
+            if x_t is None:
+                # for order 2, we use a simplified version
+                if order == 2:
+                    rhos_p = torch.tensor([0.5], device=b.device)
+                else:
+                    rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1])
+        else:
+            D1s = None
+
+        if use_corrector:
+            #print('using corrector')
+            # for order 1, we use a simplified version
+            if order == 1:
+                rhos_c = torch.tensor([0.5], device=b.device)
+            else:
+                rhos_c = torch.linalg.solve(R, b)
+
+        model_t = None
+        if self.predict_x0:
+            x_t_ = (
+                expand_dims(sigma_t / sigma_prev_0, dims) * x
+                - expand_dims(alpha_t * h_phi_1, dims)* model_prev_0
+            )
+
+            if x_t is None:
+                if use_predictor:
+                    pred_res = torch.einsum('k,bkchw->bchw', rhos_p, D1s)
+                else:
+                    pred_res = 0
+                x_t = x_t_ - expand_dims(alpha_t * B_h, dims) * pred_res
+
+            if use_corrector:
+                model_t = self.model_fn(x_t, t)
+                if D1s is not None:
+                    corr_res = torch.einsum('k,bkchw->bchw', rhos_c[:-1], D1s)
+                else:
+                    corr_res = 0
+                D1_t = (model_t - model_prev_0)
+                x_t = x_t_ - expand_dims(alpha_t * B_h, dims) * (corr_res + rhos_c[-1] * D1_t)
+        else:
+            x_t_ = (
+                expand_dims(torch.exp(log_alpha_t - log_alpha_prev_0), dims) * x
+                - expand_dims(sigma_t * h_phi_1, dims) * model_prev_0
+            )
+            if x_t is None:
+                if use_predictor:
+                    pred_res = torch.einsum('k,bkchw->bchw', rhos_p, D1s)
+                else:
+                    pred_res = 0
+                x_t = x_t_ - expand_dims(sigma_t * B_h, dims) * pred_res
+
+            if use_corrector:
+                model_t = self.model_fn(x_t, t)
+                if D1s is not None:
+                    corr_res = torch.einsum('k,bkchw->bchw', rhos_c[:-1], D1s)
+                else:
+                    corr_res = 0
+                D1_t = (model_t - model_prev_0)
+                x_t = x_t_ - expand_dims(sigma_t * B_h, dims) * (corr_res + rhos_c[-1] * D1_t)
+        return x_t, model_t
+
+
+    def sample(self, x, steps=20, t_start=None, t_end=None, order=3, skip_type='time_uniform',
+        method='singlestep', lower_order_final=True, denoise_to_zero=False, solver_type='dpm_solver',
+        atol=0.0078, rtol=0.05, corrector=False,
+    ):
+        t_0 = 1. / self.noise_schedule.total_N if t_end is None else t_end
+        t_T = self.noise_schedule.T if t_start is None else t_start
+        device = x.device
+        if method == 'multistep':
+            assert steps >= order, "UniPC order must be < sampling steps"
+            timesteps = self.get_time_steps(skip_type=skip_type, t_T=t_T, t_0=t_0, N=steps, device=device)
+            #print(f"Running UniPC Sampling with {timesteps.shape[0]} timesteps, order {order}")
+            assert timesteps.shape[0] - 1 == steps
+            with torch.no_grad():
+                vec_t = timesteps[0].expand((x.shape[0]))
+                model_prev_list = [self.model_fn(x, vec_t)]
+                t_prev_list = [vec_t]
+                with tqdm.tqdm(total=steps) as pbar:
+                    # Init the first `order` values by lower order multistep DPM-Solver.
+                    for init_order in range(1, order):
+                        vec_t = timesteps[init_order].expand(x.shape[0])
+                        x, model_x = self.multistep_uni_pc_update(x, model_prev_list, t_prev_list, vec_t, init_order, use_corrector=True)
+                        if model_x is None:
+                            model_x = self.model_fn(x, vec_t)
+                        if self.after_update is not None:
+                            self.after_update(x, model_x)
+                        model_prev_list.append(model_x)
+                        t_prev_list.append(vec_t)
+                        pbar.update()
+
+                    for step in range(order, steps + 1):
+                        vec_t = timesteps[step].expand(x.shape[0])
+                        if lower_order_final:
+                            step_order = min(order, steps + 1 - step)
+                        else:
+                            step_order = order
+                        #print('this step order:', step_order)
+                        if step == steps:
+                            #print('do not run corrector at the last step')
+                            use_corrector = False
+                        else:
+                            use_corrector = True
+                        x, model_x =  self.multistep_uni_pc_update(x, model_prev_list, t_prev_list, vec_t, step_order, use_corrector=use_corrector)
+                        if self.after_update is not None:
+                            self.after_update(x, model_x)
+                        for i in range(order - 1):
+                            t_prev_list[i] = t_prev_list[i + 1]
+                            model_prev_list[i] = model_prev_list[i + 1]
+                        t_prev_list[-1] = vec_t
+                        # We do not need to evaluate the final model value.
+                        if step < steps:
+                            if model_x is None:
+                                model_x = self.model_fn(x, vec_t)
+                            model_prev_list[-1] = model_x
+                        pbar.update()
+        else:
+            raise NotImplementedError()
+        if denoise_to_zero:
+            x = self.denoise_to_zero_fn(x, torch.ones((x.shape[0],)).to(device) * t_0)
+        return x
+
+
+#############################################################
+# other utility functions
+#############################################################
+
+def interpolate_fn(x, xp, yp):
+    """
+    A piecewise linear function y = f(x), using xp and yp as keypoints.
+    We implement f(x) in a differentiable way (i.e. applicable for autograd).
+    The function f(x) is well-defined for all x-axis. (For x beyond the bounds of xp, we use the outmost points of xp to define the linear function.)
+
+    Args:
+        x: PyTorch tensor with shape [N, C], where N is the batch size, C is the number of channels (we use C = 1 for DPM-Solver).
+        xp: PyTorch tensor with shape [C, K], where K is the number of keypoints.
+        yp: PyTorch tensor with shape [C, K].
+    Returns:
+        The function values f(x), with shape [N, C].
+    """
+    N, K = x.shape[0], xp.shape[1]
+    all_x = torch.cat([x.unsqueeze(2), xp.unsqueeze(0).repeat((N, 1, 1))], dim=2)
+    sorted_all_x, x_indices = torch.sort(all_x, dim=2)
+    x_idx = torch.argmin(x_indices, dim=2)
+    cand_start_idx = x_idx - 1
+    start_idx = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(1, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    end_idx = torch.where(torch.eq(start_idx, cand_start_idx), start_idx + 2, start_idx + 1)
+    start_x = torch.gather(sorted_all_x, dim=2, index=start_idx.unsqueeze(2)).squeeze(2)
+    end_x = torch.gather(sorted_all_x, dim=2, index=end_idx.unsqueeze(2)).squeeze(2)
+    start_idx2 = torch.where(
+        torch.eq(x_idx, 0),
+        torch.tensor(0, device=x.device),
+        torch.where(
+            torch.eq(x_idx, K), torch.tensor(K - 2, device=x.device), cand_start_idx,
+        ),
+    )
+    y_positions_expanded = yp.unsqueeze(0).expand(N, -1, -1)
+    start_y = torch.gather(y_positions_expanded, dim=2, index=start_idx2.unsqueeze(2)).squeeze(2)
+    end_y = torch.gather(y_positions_expanded, dim=2, index=(start_idx2 + 1).unsqueeze(2)).squeeze(2)
+    cand = start_y + (x - start_x) * (end_y - start_y) / (end_x - start_x)
+    return cand
+
+
+def expand_dims(v, dims):
+    """
+    Expand the tensor `v` to the dim `dims`.
+
+    Args:
+        `v`: a PyTorch tensor with shape [N].
+        `dim`: a `int`.
+    Returns:
+        a PyTorch tensor with shape [N, 1, 1, ..., 1] and the total dimension is `dims`.
+    """
+    return v[(...,) + (None,)*(dims - 1)]

modules/ngrok.py

@@ -1,25 +1,29 @@
-from pyngrok import ngrok, conf, exception
+import ngrok
 
-def connect(token, port, region):
+# Connect to ngrok for ingress
+def connect(token, port, options):
     account = None
     if token is None:
         token = 'None'
     else:
         if ':' in token:
             # token = authtoken:username:password
-            account = token.split(':')[1] + ':' + token.split(':')[-1]
-            token = token.split(':')[0]
+            token, username, password = token.split(':', 2)
+            account = f"{username}:{password}"
+
+    # For all options see: https://github.com/ngrok/ngrok-py/blob/main/examples/ngrok-connect-full.py
+    if not options.get('authtoken_from_env'):
+        options['authtoken'] = token
+    if account:
+        options['basic_auth'] = account
+    if not options.get('session_metadata'):
+        options['session_metadata'] = 'stable-diffusion-webui'
+
 
-    config = conf.PyngrokConfig(
-        auth_token=token, region=region
-    )
     try:
-        if account is None:
-            public_url = ngrok.connect(port, pyngrok_config=config, bind_tls=True).public_url
-        else:
-            public_url = ngrok.connect(port, pyngrok_config=config, bind_tls=True, auth=account).public_url
-    except exception.PyngrokNgrokError:
-        print(f'Invalid ngrok authtoken, ngrok connection aborted.\n'
+        public_url = ngrok.connect(f"127.0.0.1:{port}", **options).url()
+    except Exception as e:
+        print(f'Invalid ngrok authtoken? ngrok connection aborted due to: {e}\n'
               f'Your token: {token}, get the right one on https://dashboard.ngrok.com/get-started/your-authtoken')
     else:
         print(f'ngrok connected to localhost:{port}! URL: {public_url}\n'

modules/paths.py

@@ -1,10 +1,11 @@
-import argparse
 import os
 import sys
-import modules.safe
+from modules.paths_internal import models_path, script_path, data_path, extensions_dir, extensions_builtin_dir  # noqa: F401
 
-script_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-models_path = os.path.join(script_path, "models")
+import modules.safe  # noqa: F401
+
+
+# data_path = cmd_opts_pre.data
 sys.path.insert(0, script_path)
 
 # search for directory of stable diffusion in following places
@@ -15,11 +16,10 @@ for possible_sd_path in possible_sd_paths:
         sd_path = os.path.abspath(possible_sd_path)
         break
 
-assert sd_path is not None, "Couldn't find Stable Diffusion in any of: " + str(possible_sd_paths)
+assert sd_path is not None, f"Couldn't find Stable Diffusion in any of: {possible_sd_paths}"
 
 path_dirs = [
     (sd_path, 'ldm', 'Stable Diffusion', []),
-    (os.path.join(sd_path, '../taming-transformers'), 'taming', 'Taming Transformers', []),
     (os.path.join(sd_path, '../CodeFormer'), 'inference_codeformer.py', 'CodeFormer', []),
     (os.path.join(sd_path, '../BLIP'), 'models/blip.py', 'BLIP', []),
     (os.path.join(sd_path, '../k-diffusion'), 'k_diffusion/sampling.py', 'k_diffusion', ["atstart"]),

modules/paths_internal.py

@@ -0,0 +1,31 @@
+"""this module defines internal paths used by program and is safe to import before dependencies are installed in launch.py"""
+
+import argparse
+import os
+import sys
+import shlex
+
+commandline_args = os.environ.get('COMMANDLINE_ARGS', "")
+sys.argv += shlex.split(commandline_args)
+
+modules_path = os.path.dirname(os.path.realpath(__file__))
+script_path = os.path.dirname(modules_path)
+
+sd_configs_path = os.path.join(script_path, "configs")
+sd_default_config = os.path.join(sd_configs_path, "v1-inference.yaml")
+sd_model_file = os.path.join(script_path, 'model.ckpt')
+default_sd_model_file = sd_model_file
+
+# Parse the --data-dir flag first so we can use it as a base for our other argument default values
+parser_pre = argparse.ArgumentParser(add_help=False)
+parser_pre.add_argument("--data-dir", type=str, default=os.path.dirname(modules_path), help="base path where all user data is stored", )
+cmd_opts_pre = parser_pre.parse_known_args()[0]
+
+data_path = cmd_opts_pre.data_dir
+
+models_path = os.path.join(data_path, "models")
+extensions_dir = os.path.join(data_path, "extensions")
+extensions_builtin_dir = os.path.join(script_path, "extensions-builtin")
+config_states_dir = os.path.join(script_path, "config_states")
+
+roboto_ttf_file = os.path.join(modules_path, 'Roboto-Regular.ttf')