Compare commits
1 Commits
master
...
drew/stabl
| Author | SHA1 | Date |
|---|---|---|
 Drew Bednar
|
1bed5efdf7 | 2 years ago |
@ -1,6 +1,26 @@
|
||||
def main():
|
||||
print("Hey this is the cli application")
|
||||
import click
|
||||
|
||||
from . import run_stable_vicuna
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
"--model-dir",
|
||||
type=click.Path(exists=True, file_okay=False, dir_okay=True),
|
||||
envvar="SAVANT_MODEL_DIR",
|
||||
default="~/models",
|
||||
show_default=True,
|
||||
help="The path to a directory containing a hugging face Stable-Vicuna model.",
|
||||
)
|
||||
def stable_vicuna(model_dir):
|
||||
"""Runs a Stable Vicuna CLI prompt."""
|
||||
run_stable_vicuna.main(model_dir=model_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
cli()
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
"""
|
||||
Uses the Eleven Labs Python library and API to stream audio.
|
||||
"""
|
||||
|
||||
from elevenlabs import Voice, generate, set_api_key, stream, voices
|
||||
|
||||
from .settings import savant_settings
|
||||
|
||||
set_api_key(savant_settings.eleven_labs_api_key.get_secret_value())
|
||||
|
||||
voices = voices()
|
||||
drew_voice = voices[-1]
|
||||
drew_voice.settings.stability = 0.3
|
||||
drew_voice.settings.similarity_boost = 0.9
|
||||
|
||||
|
||||
def get_voice_model(model_name: str, voices: list[Voice]) -> Voice:
|
||||
target_voice = None
|
||||
for v in voices:
|
||||
if v.name == model_name:
|
||||
target_voice = v
|
||||
if target_voice is None:
|
||||
raise ValueError(f"Voice Model: {model_name} not found.")
|
||||
return target_voice
|
||||
|
||||
|
||||
def generate_audio(input_text: str = "") -> None:
|
||||
audio_stream = generate(text=input_text, voice=voices[-1], stream=True)
|
||||
|
||||
stream(audio_stream)
|
||||
|
||||
|
||||
def main():
|
||||
while True:
|
||||
input_text = input("Say something in Drew's voice: ")
|
||||
generate_audio(input_text=input_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,74 @@
|
||||
import textwrap
|
||||
|
||||
import colorama
|
||||
from transformers import LlamaForCausalLM, LlamaTokenizer
|
||||
from transformers import logging as t_logging
|
||||
from transformers import pipeline
|
||||
|
||||
# Configure logging level for transformers library
|
||||
t_logging.logging.set_verbosity_info()
|
||||
|
||||
|
||||
# Utility Functions
|
||||
def get_prompt(human_prompt):
|
||||
prompt_template = f"### Human: {human_prompt} \n### Assistant:"
|
||||
return prompt_template
|
||||
|
||||
|
||||
def remove_human_text(text):
|
||||
return text.split("### Human:", 1)[0]
|
||||
|
||||
|
||||
def parse_text(data):
|
||||
for item in data:
|
||||
text = item["generated_text"]
|
||||
assistant_text_index = text.find("### Assistant:")
|
||||
if assistant_text_index != -1:
|
||||
assistant_text = text[
|
||||
assistant_text_index + len("### Assistant:") :
|
||||
].strip()
|
||||
assistant_text = remove_human_text(assistant_text)
|
||||
wrapped_text = textwrap.fill(assistant_text, width=100)
|
||||
print(wrapped_text)
|
||||
|
||||
|
||||
# Reasoning question
|
||||
EXAMPLE_REASONING = "Answer the following question by reasoning step by step. \
|
||||
The cafeteria had 22 apples. If they used 20 for lunch, and bought 6 more, \
|
||||
how many apple do they have?"
|
||||
|
||||
|
||||
# User interface
|
||||
def main(model_dir):
|
||||
# Model loading for inference
|
||||
tokenizer = LlamaTokenizer.from_pretrained(model_dir)
|
||||
|
||||
base_model = LlamaForCausalLM.from_pretrained(
|
||||
model_dir,
|
||||
load_in_8bit=True,
|
||||
device_map="auto",
|
||||
)
|
||||
|
||||
pipe = pipeline(
|
||||
"text-generation",
|
||||
model=base_model,
|
||||
tokenizer=tokenizer,
|
||||
max_length=512,
|
||||
temperature=0.7,
|
||||
top_p=0.95,
|
||||
repetition_penalty=1.15,
|
||||
)
|
||||
|
||||
print("Reading for inference!")
|
||||
while True:
|
||||
input_prompt = ""
|
||||
input_prompt = input("USER:")
|
||||
print(colorama.Style.DIM + f"You are submitting: {input_prompt}")
|
||||
print(colorama.Style.RESET_ALL)
|
||||
raw_output = pipe(get_prompt(input_prompt))
|
||||
parse_text(raw_output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Warming up the engines...")
|
||||
main()
|
||||
@ -1,32 +0,0 @@
|
||||
from pydantic import BaseSettings, Field, SecretStr
|
||||
|
||||
|
||||
class SavantSettings(BaseSettings):
|
||||
"""Savant Application Settings.
|
||||
|
||||
All environment varaibles supplied should be prefixed with "SAVANT_".
|
||||
"""
|
||||
|
||||
eleven_labs_api_key: SecretStr = Field(
|
||||
default="", description="An optional Eleven Labs API key for text to speech."
|
||||
)
|
||||
eleven_labs_model: str = Field(
|
||||
default="Arnold", description="The text-to-speech model name used in eleven labs audio generation."
|
||||
)
|
||||
llm_model_name: str = Field(
|
||||
default="eachadea_vicuna-7b-1.1", description="The large language model name used in API requests."
|
||||
)
|
||||
openai_api_key: SecretStr = Field(
|
||||
default="EMPTY", description="An OPEN_API_KEY or an empty value if using FastChat replacement server"
|
||||
)
|
||||
openai_api_base: str = Field(
|
||||
default="http://localhost:8000/v1",
|
||||
description="The base url to an OpenAI API compliant endpoint. \
|
||||
Defaulted to FastChat replacement server defaults.",
|
||||
)
|
||||
|
||||
class Config:
|
||||
env_prefix = "SAVANT_"
|
||||
|
||||
|
||||
savant_settings = SavantSettings()
|
||||
@ -0,0 +1,32 @@
|
||||
# Invoke tab-completion script to be sourced with Bash shell.
|
||||
# Known to work on Bash 3.x, untested on 4.x.
|
||||
|
||||
_complete_invoke() {
|
||||
local candidates
|
||||
|
||||
# COMP_WORDS contains the entire command string up til now (including
|
||||
# program name).
|
||||
# We hand it to Invoke so it can figure out the current context: spit back
|
||||
# core options, task names, the current task's options, or some combo.
|
||||
candidates=`invoke --complete -- ${COMP_WORDS[*]}`
|
||||
|
||||
# `compgen -W` takes list of valid options & a partial word & spits back
|
||||
# possible matches. Necessary for any partial word completions (vs
|
||||
# completions performed when no partial words are present).
|
||||
#
|
||||
# $2 is the current word or token being tabbed on, either empty string or a
|
||||
# partial word, and thus wants to be compgen'd to arrive at some subset of
|
||||
# our candidate list which actually matches.
|
||||
#
|
||||
# COMPREPLY is the list of valid completions handed back to `complete`.
|
||||
COMPREPLY=( $(compgen -W "${candidates}" -- $2) )
|
||||
}
|
||||
|
||||
|
||||
# Tell shell builtin to use the above for completing our invocations.
|
||||
# * -F: use given function name to generate completions.
|
||||
# * -o default: when function generates no results, use filenames.
|
||||
# * positional args: program names to complete for.
|
||||
complete -F _complete_invoke -o default invoke inv
|
||||
|
||||
# vim: set ft=sh :
|
||||
@ -1 +1,9 @@
|
||||
pydantic>=1.6.2,<2.0.0
|
||||
transformers @ git+https://github.com/huggingface/transformers@849367ccf741d8c58aa88ccfe1d52d8636eaf2b7
|
||||
bitsandbytes
|
||||
datasets
|
||||
loralib
|
||||
sentencepiece
|
||||
bitsandbytes
|
||||
accelerate
|
||||
langchain
|
||||
colorama
|
||||
|
||||
@ -1,33 +0,0 @@
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from pydantic.types import SecretStr
|
||||
|
||||
from chat_savant.settings import SavantSettings
|
||||
|
||||
SETTING_DEFAULTS = {
|
||||
"OPENAI_API_KEY": "EMPTY",
|
||||
"OPENAI_API_BASE": "http://localhost:8000/v1",
|
||||
"LLM_MODEL_NAME": "eachadea_vicuna-7b-1.1",
|
||||
"ELEVEN_LABS_API_KEY": "",
|
||||
"ELEVEN_LABS_MODEL": "",
|
||||
}
|
||||
|
||||
|
||||
def test_setting_defaults():
|
||||
"""Regression test for settings schema."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
savant_settings = SavantSettings()
|
||||
assert len(savant_settings.dict()) == len(SETTING_DEFAULTS)
|
||||
for k, v in SETTING_DEFAULTS.items():
|
||||
_setting_value = getattr(savant_settings, k.lower())
|
||||
unmasked_setting = (
|
||||
_setting_value.get_secret_value() if isinstance(_setting_value, SecretStr) else _setting_value
|
||||
)
|
||||
unmasked_setting == v
|
||||
|
||||
|
||||
def test_with_envvar_prefix():
|
||||
with patch.dict(os.environ, {"SAVANT_ELEVEN_LABS_API_KEY": "thisisnotreal"}, clear=True):
|
||||
savant_settings = SavantSettings()
|
||||
assert savant_settings.eleven_labs_api_key.get_secret_value() == "thisisnotreal"
|
||||
Loading…
Reference in New Issue