From a80a5465f269d3bf951478517cfc1236c73519ec Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 17:27:29 -0500 Subject: [PATCH 01/18] Update install.bat Updated Conda packages and channels to install cuda-toolkit and override 12.0 cuda packages requested by pytorch with their 11.7 equivalent. Removed Conda installation since we can use the downloaded Micromamba.exe for the same purpose with a smaller footprint. Removed redundant PATH changes. Changed %gpuchoice% comparisons to be case-insensitive. Added additional error handling and removed the use of .tmp files. Added missing extension requirements. Added GPTQ installation. Will attempt to compile locally and, if failed, will download and install a precompiled wheel. Incorporated fixes from one-click-bandaid. Fixed and expanded first sed command from one-click-bandaid. libbitsandbytes_cudaall.dll is used here as the cuda116.dll used by one-click-bandaid does not work on my 1080ti. This can be changed if needed. --- install.bat | 93 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/install.bat b/install.bat index 4dc7018..58875ea 100644 --- a/install.bat +++ b/install.bat @@ -1,7 +1,7 @@ @echo off @rem Based on the installer found here: https://github.com/Sygil-Dev/sygil-webui -@rem This script will install git and conda (if not found on the PATH variable) +@rem This script will install git and all dependencies @rem using micromamba (an 8mb static-linked single-file binary, conda replacement). @rem This enables a user to install this project without manually installing conda and git. @@ -12,16 +12,13 @@ echo B) None (I want to run in CPU mode) echo. set /p "gpuchoice=Input> " set gpuchoice=%gpuchoice:~0,1% -setlocal enabledelayedexpansion -set gpuchoice=!gpuchoice:a=A! -set gpuchoice=!gpuchoice:b=B! -if "%gpuchoice%" == "A" ( - set "PACKAGES_TO_INSTALL=torchvision=0.14.1 torchaudio=0.13.1 pytorch-cuda=11.7 conda git" - set "CHANNEL=-c nvidia" -) else if "%gpuchoice%" == "B" ( - set "PACKAGES_TO_INSTALL=pytorch torchvision torchaudio cpuonly conda git" - set "CHANNEL=" +if /I "%gpuchoice%" == "A" ( + set "PACKAGES_TO_INSTALL=python=3.10.9 pytorch torchvision torchaudio pytorch-cuda=11.7 cuda-toolkit conda-forge::ninja conda-forge::git" + set "CHANNEL=-c pytorch -c nvidia/label/cuda-11.7.0 -c nvidia" +) else if /I "%gpuchoice%" == "B" ( + set "PACKAGES_TO_INSTALL=pytorch torchvision torchaudio cpuonly git" + set "CHANNEL=-c conda-forge -c pytorch" ) else ( echo Invalid choice. Exiting... exit @@ -34,57 +31,91 @@ set REPO_URL=https://github.com/oobabooga/text-generation-webui.git set umamba_exists=F @rem figure out whether git and conda needs to be installed -if exist "%INSTALL_ENV_DIR%" set PATH=%INSTALL_ENV_DIR%;%INSTALL_ENV_DIR%\Library\bin;%INSTALL_ENV_DIR%\Scripts;%INSTALL_ENV_DIR%\Library\usr\bin;%PATH% -call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version >.tmp1 2>.tmp2 +call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version >nul 2>&1 if "%ERRORLEVEL%" EQU "0" set umamba_exists=T @rem (if necessary) install git and conda into a contained environment if "%PACKAGES_TO_INSTALL%" NEQ "" ( @rem download micromamba if "%umamba_exists%" == "F" ( - echo "Downloading micromamba from %MICROMAMBA_DOWNLOAD_URL% to %MAMBA_ROOT_PREFIX%\micromamba.exe" + echo "Downloading Micromamba from %MICROMAMBA_DOWNLOAD_URL% to %MAMBA_ROOT_PREFIX%\micromamba.exe" mkdir "%MAMBA_ROOT_PREFIX%" call curl -L "%MICROMAMBA_DOWNLOAD_URL%" > "%MAMBA_ROOT_PREFIX%\micromamba.exe" @rem test the mamba binary echo Micromamba version: - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" --version || ( echo Micromamba not found. && goto end ) ) + @rem create micromamba hook + if not exist "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" ( + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 + ) + + @rem activate base micromamba env + call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) + @rem create the installer env if not exist "%INSTALL_ENV_DIR%" ( - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" create -y --prefix "%INSTALL_ENV_DIR%" + call micromamba create -y --prefix "%INSTALL_ENV_DIR%" ) + @rem activate installer env + call micromamba activate "%INSTALL_ENV_DIR%" || ( echo %INSTALL_ENV_DIR% not found. && goto end ) echo "Packages to install: %PACKAGES_TO_INSTALL%" - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" install -y --prefix "%INSTALL_ENV_DIR%" -c conda-forge -c pytorch %CHANNEL% %PACKAGES_TO_INSTALL% - call "%MAMBA_ROOT_PREFIX%\micromamba.exe" install -y --prefix "%INSTALL_ENV_DIR%" -c conda-forge -c pytorch %CHANNEL% %PACKAGES_TO_INSTALL% - - if not exist "%INSTALL_ENV_DIR%" ( - echo "There was a problem while installing%PACKAGES_TO_INSTALL% using micromamba. Cannot continue." - pause - exit /b - ) + call micromamba install -y %CHANNEL% %PACKAGES_TO_INSTALL% ) -set PATH=%INSTALL_ENV_DIR%;%INSTALL_ENV_DIR%\Library\bin;%INSTALL_ENV_DIR%\Scripts;%INSTALL_ENV_DIR%\Library\usr\bin;%PATH% - @rem clone the repository and install the pip requirements -call conda activate if exist text-generation-webui\ ( cd text-generation-webui git pull ) else ( git clone https://github.com/oobabooga/text-generation-webui.git - cd text-generation-webui + cd text-generation-webui || goto end ) call python -m pip install -r requirements.txt --upgrade -call python -m pip install -r extensions\google_translate\requirements.txt -call python -m pip install -r extensions\silero_tts\requirements.txt +call python -m pip install -r extensions\api\requirements.txt --upgrade +call python -m pip install -r extensions\elevenlabs_tts\requirements.txt --upgrade +call python -m pip install -r extensions\google_translate\requirements.txt --upgrade +call python -m pip install -r extensions\silero_tts\requirements.txt --upgrade +call python -m pip install -r extensions\whisper_stt\requirements.txt --upgrade -cd .. -del .tmp1 .tmp2 +@rem skip gptq install if cpu only +if /I not "%gpuchoice%" == "A" goto bandaid +@rem download gptq and compile locally and if compile fails, install from wheel +if not exist repositories\ ( + mkdir repositories +) +cd repositories || goto end +if not exist GPTQ-for-LLaMa\ ( + git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git + cd GPTQ-for-LLaMa || goto end + git reset --hard 468c47c01b4fe370616747b6d69a2d3f48bab5e4 + pip install -r requirements.txt + call python setup_cuda.py install + if not exist "%INSTALL_ENV_DIR%\lib\site-packages\quant_cuda-0.0.0-py3.10-win-amd64.egg" ( + echo CUDA kernal compilation failed. Will try to install from wheel. + pip install unzip + curl -LO https://github.com/oobabooga/text-generation-webui/files/11023775/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip + unzip quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip + pip install quant_cuda-0.0.0-cp310-cp310-win_amd64.whl || ( echo Wheel installation failed. && goto end ) + ) + cd .. +) +cd ..\.. + +:bandaid +curl -LO https://github.com/DeXtmL/bitsandbytes-win-prebuilt/raw/main/libbitsandbytes_cpu.dll +curl -LO https://github.com/james-things/bitsandbytes-prebuilt-all_arch/raw/main/0.37.0/libbitsandbytes_cudaall.dll +mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" +mv libbitsandbytes_cuda116.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" +pip install sed +sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" +sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" + +:end pause From 817e6c681e98b6cf48df42c37dbfebdd715a21a8 Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 17:51:13 -0500 Subject: [PATCH 02/18] Update install.bat Added `cd /D "%~dp0"` in case the script is ran as admin. --- install.bat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install.bat b/install.bat index 58875ea..b7f921d 100644 --- a/install.bat +++ b/install.bat @@ -24,6 +24,8 @@ if /I "%gpuchoice%" == "A" ( exit ) +cd /D "%~dp0" + set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env set MICROMAMBA_DOWNLOAD_URL=https://github.com/cmdr2/stable-diffusion-ui/releases/download/v1.1/micromamba.exe From eec773b1f4c8a7dbb090d59a44ef788ab7cc9a59 Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 17:54:47 -0500 Subject: [PATCH 03/18] Update install.bat Corrected libbitsandbytes_cudaall.dll install. --- install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.bat b/install.bat index b7f921d..e0ffa7b 100644 --- a/install.bat +++ b/install.bat @@ -114,7 +114,7 @@ cd ..\.. curl -LO https://github.com/DeXtmL/bitsandbytes-win-prebuilt/raw/main/libbitsandbytes_cpu.dll curl -LO https://github.com/james-things/bitsandbytes-prebuilt-all_arch/raw/main/0.37.0/libbitsandbytes_cudaall.dll mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" -mv libbitsandbytes_cuda116.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" +mv libbitsandbytes_cudaall.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" pip install sed sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" From f0c82f06c364c5a3dfc9088100f537ec3d9f987a Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:09:44 -0500 Subject: [PATCH 04/18] Add files via upload Add script to open cmd within installation environment for easier modification. --- micromamba-cmd.bat | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 micromamba-cmd.bat diff --git a/micromamba-cmd.bat b/micromamba-cmd.bat new file mode 100644 index 0000000..de868fd --- /dev/null +++ b/micromamba-cmd.bat @@ -0,0 +1,15 @@ +@echo off + +set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba +set INSTALL_ENV_DIR=%cd%\installer_files\env + +if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 +) +call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) +call micromamba activate "%INSTALL_ENV_DIR%" || goto end + +cmd /k "%*" + +:end +pause \ No newline at end of file From 24870e51ed6a53d81ddb33970baae0c7c1aa7a69 Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:12:02 -0500 Subject: [PATCH 05/18] Update micromamba-cmd.bat Add cd command for admin. --- micromamba-cmd.bat | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/micromamba-cmd.bat b/micromamba-cmd.bat index de868fd..0365fce 100644 --- a/micromamba-cmd.bat +++ b/micromamba-cmd.bat @@ -1,5 +1,7 @@ @echo off +cd /D "%~dp0" + set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env @@ -12,4 +14,4 @@ call micromamba activate "%INSTALL_ENV_DIR%" || goto end cmd /k "%*" :end -pause \ No newline at end of file +pause From 2604e3f7ac580eeabd548d3a152ba375c61c279c Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:15:24 -0500 Subject: [PATCH 06/18] Update download-model.bat Added variables for model selection and text only mode. Updated virtual environment handling to use Micromamba. --- download-model.bat | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/download-model.bat b/download-model.bat index 874d359..148f493 100644 --- a/download-model.bat +++ b/download-model.bat @@ -1,9 +1,28 @@ @echo off -set INSTALL_ENV_DIR=%cd%\installer_files\env -set PATH=%INSTALL_ENV_DIR%;%INSTALL_ENV_DIR%\Library\bin;%INSTALL_ENV_DIR%\Scripts;%INSTALL_ENV_DIR%\Library\usr\bin;%PATH% -call conda activate -cd text-generation-webui -python download-model.py +SET TextOnly=False &REM True or False for Text only mode +SET ModelName="chansung/alpaca-lora-13b" &REM HuggingFace model repo eg. "facebook/opt-1.3b" +cd /D "%~dp0" + +set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba +set INSTALL_ENV_DIR=%cd%\installer_files\env + +if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 +) +call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo MicroMamba hook not found. && goto end ) +call micromamba activate "%INSTALL_ENV_DIR%" || goto end + +cd text-generation-webui || goto end +goto %TextOnly% + +:False +call python download-model.py %ModelName% +goto end + +:True +call python download-model.py %ModelName% --text-only + +:end pause From bddbc2f89848d4cbc554c8f91a39761955b34b76 Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:19:23 -0500 Subject: [PATCH 07/18] Update start-webui.bat Updated virtual environment handling to use Micromamba. --- start-webui.bat | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/start-webui.bat b/start-webui.bat index 2a4189d..32d7833 100644 --- a/start-webui.bat +++ b/start-webui.bat @@ -2,10 +2,19 @@ @echo Starting the web UI... +cd /D "%~dp0" + +set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env -set PATH=%INSTALL_ENV_DIR%;%INSTALL_ENV_DIR%\Library\bin;%INSTALL_ENV_DIR%\Scripts;%INSTALL_ENV_DIR%\Library\usr\bin;%PATH% -call conda activate + +if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 +) +call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) +call micromamba activate "%INSTALL_ENV_DIR%" || goto end cd text-generation-webui + call python server.py --auto-devices --cai-chat +:end pause From 586775ad47712c4e7117a48dbcdb3f582db57d8f Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:25:49 -0500 Subject: [PATCH 08/18] Update download-model.bat Removed redundant %ModelName% variable. --- download-model.bat | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/download-model.bat b/download-model.bat index 148f493..e1b7442 100644 --- a/download-model.bat +++ b/download-model.bat @@ -1,7 +1,6 @@ @echo off SET TextOnly=False &REM True or False for Text only mode -SET ModelName="chansung/alpaca-lora-13b" &REM HuggingFace model repo eg. "facebook/opt-1.3b" cd /D "%~dp0" @@ -18,11 +17,11 @@ cd text-generation-webui || goto end goto %TextOnly% :False -call python download-model.py %ModelName% +call python download-model.py goto end :True -call python download-model.py %ModelName% --text-only +call python download-model.py --text-only :end pause From fa916aa1de89b068cece26c45e3da255f6c8d021 Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 18:28:46 -0500 Subject: [PATCH 09/18] Update INSTRUCTIONS.txt Added clarification on new variable added to download-model.bat. --- INSTRUCTIONS.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTRUCTIONS.txt b/INSTRUCTIONS.txt index 9afcc4a..668d906 100644 --- a/INSTRUCTIONS.txt +++ b/INSTRUCTIONS.txt @@ -2,7 +2,7 @@ Thank you for downloading oobabooga/text-generation-webui. Here is how to get it up and running: 1. Run the "install" script to install the web UI and its requirements in this folder. -2. Run the "download" script to download a model of your choice. +2. Run the "download" script to download a model of your choice. Change TextOnly variable at top of script to download only config files. 3. Run the "start-webui" script to launch the web UI. To add flags like --chat, --notebook, --extensions, etc, edit the From 1e260544cd41023566a32fa297c89fb91d22948c Mon Sep 17 00:00:00 2001 From: jllllll Date: Fri, 24 Mar 2023 21:25:14 -0500 Subject: [PATCH 10/18] Update install.bat Added C:\Windows\System32 to PATH to avoid issues with broken? Windows installs. --- install.bat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install.bat b/install.bat index e0ffa7b..ac86f77 100644 --- a/install.bat +++ b/install.bat @@ -26,6 +26,8 @@ if /I "%gpuchoice%" == "A" ( cd /D "%~dp0" +set PATH=%SystemRoot%\system32;%PATH% + set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env set MICROMAMBA_DOWNLOAD_URL=https://github.com/cmdr2/stable-diffusion-ui/releases/download/v1.1/micromamba.exe From 2e02d42682e9a6765fb16843d516efd235537484 Mon Sep 17 00:00:00 2001 From: jllllll Date: Sat, 25 Mar 2023 01:14:29 -0500 Subject: [PATCH 11/18] Changed things around to allow Micromamba to work with paths containing spaces. --- download-model.bat | 5 ++--- install.bat | 19 +++++++------------ micromamba-cmd.bat | 5 ++--- start-webui.bat | 5 ++--- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/download-model.bat b/download-model.bat index e1b7442..af2610c 100644 --- a/download-model.bat +++ b/download-model.bat @@ -7,11 +7,10 @@ cd /D "%~dp0" set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env -if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( +if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 ) -call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo MicroMamba hook not found. && goto end ) -call micromamba activate "%INSTALL_ENV_DIR%" || goto end +call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) cd text-generation-webui || goto end goto %TextOnly% diff --git a/install.bat b/install.bat index ac86f77..43c6894 100644 --- a/install.bat +++ b/install.bat @@ -53,25 +53,20 @@ if "%PACKAGES_TO_INSTALL%" NEQ "" ( ) @rem create micromamba hook - if not exist "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" ( + if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 ) - @rem activate base micromamba env - call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) - @rem create the installer env if not exist "%INSTALL_ENV_DIR%" ( - call micromamba create -y --prefix "%INSTALL_ENV_DIR%" + echo Packages to install: %PACKAGES_TO_INSTALL% + call "%MAMBA_ROOT_PREFIX%\micromamba.exe" create -y --prefix "%INSTALL_ENV_DIR%" %CHANNEL% %PACKAGES_TO_INSTALL% ) - @rem activate installer env - call micromamba activate "%INSTALL_ENV_DIR%" || ( echo %INSTALL_ENV_DIR% not found. && goto end ) - - echo "Packages to install: %PACKAGES_TO_INSTALL%" - - call micromamba install -y %CHANNEL% %PACKAGES_TO_INSTALL% ) +@rem activate installer env +call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) + @rem clone the repository and install the pip requirements if exist text-generation-webui\ ( cd text-generation-webui @@ -122,4 +117,4 @@ sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', No sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" :end -pause +pause \ No newline at end of file diff --git a/micromamba-cmd.bat b/micromamba-cmd.bat index 0365fce..355e7b4 100644 --- a/micromamba-cmd.bat +++ b/micromamba-cmd.bat @@ -5,11 +5,10 @@ cd /D "%~dp0" set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env -if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( +if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 ) -call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) -call micromamba activate "%INSTALL_ENV_DIR%" || goto end +call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) cmd /k "%*" diff --git a/start-webui.bat b/start-webui.bat index 32d7833..694f07a 100644 --- a/start-webui.bat +++ b/start-webui.bat @@ -7,11 +7,10 @@ cd /D "%~dp0" set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env -if not exist "%MAMBA_ROOT_PREFIX%\Scripts\activate.bat" ( +if not exist "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" ( call "%MAMBA_ROOT_PREFIX%\micromamba.exe" shell hook >nul 2>&1 ) -call "%MAMBA_ROOT_PREFIX%\condabin\mamba_hook.bat" || ( echo Micromamba hook not found. && goto end ) -call micromamba activate "%INSTALL_ENV_DIR%" || goto end +call "%MAMBA_ROOT_PREFIX%\condabin\micromamba.bat" activate "%INSTALL_ENV_DIR%" || ( echo MicroMamba hook not found. && goto end ) cd text-generation-webui call python server.py --auto-devices --cai-chat From ce9a5e3b531d223b8e073ce4ec8a3858747aa9ff Mon Sep 17 00:00:00 2001 From: jllllll Date: Sat, 25 Mar 2023 02:22:02 -0500 Subject: [PATCH 12/18] Update install.bat Minor fixes --- install.bat | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/install.bat b/install.bat index 43c6894..47eba25 100644 --- a/install.bat +++ b/install.bat @@ -94,14 +94,14 @@ if not exist GPTQ-for-LLaMa\ ( git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git cd GPTQ-for-LLaMa || goto end git reset --hard 468c47c01b4fe370616747b6d69a2d3f48bab5e4 - pip install -r requirements.txt + call python -m pip install -r requirements.txt call python setup_cuda.py install if not exist "%INSTALL_ENV_DIR%\lib\site-packages\quant_cuda-0.0.0-py3.10-win-amd64.egg" ( echo CUDA kernal compilation failed. Will try to install from wheel. - pip install unzip + call python -m pip install unzip curl -LO https://github.com/oobabooga/text-generation-webui/files/11023775/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip unzip quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip - pip install quant_cuda-0.0.0-cp310-cp310-win_amd64.whl || ( echo Wheel installation failed. && goto end ) + call python -m pip install quant_cuda-0.0.0-cp310-cp310-win_amd64.whl || ( echo Wheel installation failed. && goto end ) ) cd .. ) @@ -112,9 +112,8 @@ curl -LO https://github.com/DeXtmL/bitsandbytes-win-prebuilt/raw/main/libbitsand curl -LO https://github.com/james-things/bitsandbytes-prebuilt-all_arch/raw/main/0.37.0/libbitsandbytes_cudaall.dll mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" mv libbitsandbytes_cudaall.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" -pip install sed sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" :end -pause \ No newline at end of file +pause From 247e8e5b798e5cd8c9cc2ddf7974b97a7359b338 Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 00:24:00 -0500 Subject: [PATCH 13/18] Fix for issue in current GPTQ-for-LLaMa. --- install.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/install.bat b/install.bat index 47eba25..f1f70bf 100644 --- a/install.bat +++ b/install.bat @@ -114,6 +114,7 @@ mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" mv libbitsandbytes_cudaall.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" +sed -i "s/make_quant(model, layers, wbits, groupsize, faster=args.faster_kernel)/make_quant(model, layers, wbits, groupsize)/g" "%INSTALL_ENV_DIR%\..\..\text-generation-webui\repositories\GPTQ-for-LLaMa\llama.py" :end pause From 12baa0e84b04d37b61a7ac120d4f1396a89c6cbd Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 00:46:07 -0500 Subject: [PATCH 14/18] Update for latest GPTQ-for-LLaMa --- install.bat | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/install.bat b/install.bat index f1f70bf..46b108a 100644 --- a/install.bat +++ b/install.bat @@ -93,14 +93,11 @@ cd repositories || goto end if not exist GPTQ-for-LLaMa\ ( git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git cd GPTQ-for-LLaMa || goto end - git reset --hard 468c47c01b4fe370616747b6d69a2d3f48bab5e4 call python -m pip install -r requirements.txt call python setup_cuda.py install if not exist "%INSTALL_ENV_DIR%\lib\site-packages\quant_cuda-0.0.0-py3.10-win-amd64.egg" ( echo CUDA kernal compilation failed. Will try to install from wheel. - call python -m pip install unzip - curl -LO https://github.com/oobabooga/text-generation-webui/files/11023775/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip - unzip quant_cuda-0.0.0-cp310-cp310-win_amd64.whl.zip + curl -LO https://github.com/jllllll/GPTQ-for-LLaMa-Wheels/raw/main/quant_cuda-0.0.0-cp310-cp310-win_amd64.whl call python -m pip install quant_cuda-0.0.0-cp310-cp310-win_amd64.whl || ( echo Wheel installation failed. && goto end ) ) cd .. From 6dcfcf4fed425f1574d65becf7001a779530946a Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 01:00:52 -0500 Subject: [PATCH 15/18] Amended fix for GPTQ-for-LLaMa Prevents breaking 3-bit support --- install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.bat b/install.bat index 46b108a..9f7f5ca 100644 --- a/install.bat +++ b/install.bat @@ -111,7 +111,7 @@ mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" mv libbitsandbytes_cudaall.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" -sed -i "s/make_quant(model, layers, wbits, groupsize, faster=args.faster_kernel)/make_quant(model, layers, wbits, groupsize)/g" "%INSTALL_ENV_DIR%\..\..\text-generation-webui\repositories\GPTQ-for-LLaMa\llama.py" +sed -i "s/make_quant(model, layers, wbits, groupsize, faster=args.faster_kernel)/make_quant(model, layers, wbits, groupsize, faster=("args" in globals() and args.faster_kernel))/g" "%INSTALL_ENV_DIR%\..\..\text-generation-webui\repositories\GPTQ-for-LLaMa\llama.py" :end pause From 6f892420949882aae2d123d87d3eccb5843ccee0 Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 03:29:14 -0500 Subject: [PATCH 16/18] Remove temporary fix for GPTQ-for-LLaMa No longer necessary. --- install.bat | 1 - 1 file changed, 1 deletion(-) diff --git a/install.bat b/install.bat index 9f7f5ca..926971f 100644 --- a/install.bat +++ b/install.bat @@ -111,7 +111,6 @@ mv libbitsandbytes_cpu.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" mv libbitsandbytes_cudaall.dll "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes" sed -i "s/if not torch.cuda.is_available(): return 'libsbitsandbytes_cpu.so', None, None, None, None/if torch.cuda.is_available(): return 'libbitsandbytes_cudaall.dll', None, None, None, None\n else: return 'libbitsandbytes_cpu.dll', None, None, None, None/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" sed -i "s/ct.cdll.LoadLibrary(binary_path)/ct.cdll.LoadLibrary(str(binary_path))/g" "%INSTALL_ENV_DIR%\lib\site-packages\bitsandbytes\cuda_setup\main.py" -sed -i "s/make_quant(model, layers, wbits, groupsize, faster=args.faster_kernel)/make_quant(model, layers, wbits, groupsize, faster=("args" in globals() and args.faster_kernel))/g" "%INSTALL_ENV_DIR%\..\..\text-generation-webui\repositories\GPTQ-for-LLaMa\llama.py" :end pause From bdf85ffcf97f2e0f7452936f7fef56e68a076a10 Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 21:56:16 -0500 Subject: [PATCH 17/18] Remove explicit pytorch installation Fixes an issue some people were having: https://github.com/oobabooga/text-generation-webui/issues/15 I did not experience this issue on my system. Not everyone does for some reason. --- install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.bat b/install.bat index 926971f..68e1b72 100644 --- a/install.bat +++ b/install.bat @@ -14,7 +14,7 @@ set /p "gpuchoice=Input> " set gpuchoice=%gpuchoice:~0,1% if /I "%gpuchoice%" == "A" ( - set "PACKAGES_TO_INSTALL=python=3.10.9 pytorch torchvision torchaudio pytorch-cuda=11.7 cuda-toolkit conda-forge::ninja conda-forge::git" + set "PACKAGES_TO_INSTALL=python=3.10.9 torchvision torchaudio pytorch-cuda=11.7 cuda-toolkit conda-forge::ninja conda-forge::git" set "CHANNEL=-c pytorch -c nvidia/label/cuda-11.7.0 -c nvidia" ) else if /I "%gpuchoice%" == "B" ( set "PACKAGES_TO_INSTALL=pytorch torchvision torchaudio cpuonly git" From cb5dff0087122f62a1aa9bcc8a2c24851c6df3f3 Mon Sep 17 00:00:00 2001 From: jllllll Date: Sun, 26 Mar 2023 23:40:46 -0500 Subject: [PATCH 18/18] Update installer to use official micromamba url --- install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.bat b/install.bat index 68e1b72..914cd7e 100644 --- a/install.bat +++ b/install.bat @@ -30,7 +30,7 @@ set PATH=%SystemRoot%\system32;%PATH% set MAMBA_ROOT_PREFIX=%cd%\installer_files\mamba set INSTALL_ENV_DIR=%cd%\installer_files\env -set MICROMAMBA_DOWNLOAD_URL=https://github.com/cmdr2/stable-diffusion-ui/releases/download/v1.1/micromamba.exe +set MICROMAMBA_DOWNLOAD_URL=https://github.com/mamba-org/micromamba-releases/releases/latest/download/micromamba-win-64 set REPO_URL=https://github.com/oobabooga/text-generation-webui.git set umamba_exists=F